The truly free, open-source, local-first, cross-platform CapCut replacement. Scriptable through a JSON-RPC API & MCP.
Rust ffmpegfree-video-editormodel-context-protocolnon-linear-editoroffline-firstopen-source-video-editorself-hostedvideo-editingvideo-editing-automationvideo-editing-softwarevideo-editing-toolsvideo-editorvoice-cloningyoutube-automationyoutube-shorts
Composite the monitor on the window's own GPU device
Jareer7ee00f9ca571
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.mdindex be7af841..013673db 100644--- a/ARCHITECTURE.md+++ b/ARCHITECTURE.md@@ -213,9 +213,10 @@ flowchart TBendsubgraph VIDEO["Video — concat_host::preview::Monitor"]PULL["window pulls one frame per instant<br/>prefetch warms the next few"]- PF["preview_frame → raw RGBA"]- IMG["slint::Image from the buffer"]- PULL --> PF --> IMG+ PF["preview_sources → decoded layers"]+ TEX["WgpuCompositor on the shared device → texture"]+ IMG["slint::Image::try_from(texture)"]+ PULL --> PF --> TEX --> IMGendCLK -- "the window follows" --> PULL```@@ -225,6 +226,12 @@ flowchart TB30 Hz while playing. Video chases audio, never the other way around.- **The monitor shows the engine's real composited frame** - the same pixelsthe exporter would produce - at a preview size the window chooses.+- **One GPU device, shared.** The window creates a wgpu device (`gpu.rs`)+ and hands it to Slint's renderer and to the monitor's compositor. A+ preview frame is decoded, uploaded once per layer, composited on that+ device into a presentable texture, and shown by Slint as that texture:+ nothing is read back and nothing is copied. Without an adapter the+ monitor composites on the CPU and the window uploads the pixels.- **Nothing here knows about a window.** `Playback` reports through a traitobject; `Monitor` returns bytes. The window decides which thread runs what,and hands results back to the Slint event loop withdiff --git a/engine/Cargo.lock b/engine/Cargo.lockindex c1d1a974..54a423c9 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -230,6 +230,15 @@ dependencies = ["thiserror 2.0.20",]+[[package]]+name = "android-build"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f9fc9904ad2ad097c3c1cfe2eacaaf0fc24710936fa9ed941cb310b7c6ed2ab7"+dependencies = [+ "windows-sys 0.52.0",+]+[[package]]name = "android-properties"version = "0.2.2"@@ -648,15 +657,6 @@ dependencies = ["bit-vec",]-[[package]]-name = "bit-set"-version = "0.10.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "09ec2f926cc3060f09db9ebc5b52823d85268d24bb917e472c0c4bea35780a7d"-dependencies = [- "bit-vec",-]-[[package]]name = "bit-vec"version = "0.9.1"@@ -1102,11 +1102,13 @@ dependencies = ["concat-project","concat-speech","opener",+ "pollster","rfd","serde","serde_json","slint","slint-build",+ "wgpu",][[package]]@@ -1153,6 +1155,7 @@ dependencies = ["concat-export","concat-media","concat-project",+ "concat-render","cpal","memmap2","serde",@@ -1184,7 +1187,7 @@ version = "0.2.0"dependencies = ["concat-core","pollster",- "wgpu 30.0.1",+ "wgpu",][[package]]@@ -1805,7 +1808,7 @@ dependencies = ["swash","wasm-bindgen","web-sys",- "wgpu 29.0.4",+ "wgpu",][[package]]@@ -2436,6 +2439,20 @@ version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "48ce8546b993eaf241d69ded33b1be6d205dd9857ec879d9d18bd05d3676e144"+[[package]]+name = "i-slint-backend-android-activity"+version = "1.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9805bcdcbeea6626a0054347ee55d2424e0d5574b563472719d3fc7a11852242"+dependencies = [+ "android-build",+ "i-slint-common",+ "i-slint-core",+ "i-slint-renderer-skia",+ "jni 0.22.4",+ "raw-window-handle",+]+[[package]]name = "i-slint-backend-linuxkms"version = "1.17.1"@@ -2639,7 +2656,7 @@ dependencies = ["wasm-bindgen","web-sys","web-time",- "wgpu 29.0.4",+ "wgpu","windows 0.62.2",]@@ -2674,7 +2691,7 @@ dependencies = ["rgb","wasm-bindgen","web-sys",- "wgpu 29.0.4",+ "wgpu",][[package]]@@ -2712,7 +2729,7 @@ dependencies = ["spin_on","unicode-segmentation","vtable",- "wgpu 29.0.4",+ "wgpu","windows 0.62.2","windows-core 0.62.2","write-fonts",@@ -3565,7 +3582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b2bf919621e7975acb27d881bae2fb993e0d45c8e0446e85e6272971e00dc8df"dependencies = ["arrayvec",- "bit-set 0.9.1",+ "bit-set","bitflags 2.13.1","cfg-if","cfg_aliases",@@ -3584,44 +3601,6 @@ dependencies = ["unicode-ident",]-[[package]]-name = "naga"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a616d2fb8c89516ac2723a581f69d6c18576046bed761bd6b305e5618e6ae130"-dependencies = [- "arrayvec",- "bit-set 0.10.0",- "bitflags 2.13.1",- "cfg-if",- "cfg_aliases",- "codespan-reporting",- "half",- "hashbrown 0.17.1",- "indexmap",- "libm",- "log",- "naga-types",- "num-traits",- "once_cell",- "rustc-hash 1.1.0",- "spirv",- "thiserror 2.0.20",- "unicode-ident",-]--[[package]]-name = "naga-types"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "590afbf58a6f4f62873cd5cff4468061844bafa1cdf399cc954537c22d768d49"-dependencies = [- "hashbrown 0.17.1",- "indexmap",- "rustc-hash 1.1.0",- "thiserror 2.0.20",-]-[[package]]name = "natord"version = "1.0.9"@@ -4148,7 +4127,6 @@ dependencies = ["bitflags 2.13.1","objc2 0.6.4","objc2-core-foundation",- "objc2-core-graphics","objc2-foundation 0.3.2","objc2-metal 0.3.2",]@@ -5427,6 +5405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ce12038f57f8f3a423564b0c9301a2ab3fda4246e4cbadd6b97acf9124bdc315"dependencies = ["const-field-offset",+ "i-slint-backend-android-activity","i-slint-backend-selector","i-slint-backend-winit","i-slint-common",@@ -5441,6 +5420,7 @@ dependencies = ["slint-macros","unicode-segmentation","vtable",+ "wgpu",][[package]]@@ -6611,37 +6591,7 @@ dependencies = ["hashbrown 0.16.1","js-sys","log",- "naga 29.0.4",- "parking_lot",- "portable-atomic",- "profiling",- "raw-window-handle",- "smallvec",- "static_assertions",- "wasm-bindgen",- "wasm-bindgen-futures",- "web-sys",- "wgpu-core 29.0.4",- "wgpu-hal 29.0.4",- "wgpu-types 29.0.4",-]--[[package]]-name = "wgpu"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "527ccdf43dd5b2e8676eed9984ce00e2bbb0a1b85b70c1969dcb6cd2eb55ab9e"-dependencies = [- "arrayvec",- "bitflags 2.13.1",- "bytemuck",- "cfg-if",- "cfg_aliases",- "document-features",- "hashbrown 0.17.1",- "js-sys",- "log",- "naga 30.0.1",+ "naga","parking_lot","portable-atomic","profiling",@@ -6651,9 +6601,9 @@ dependencies = ["wasm-bindgen","wasm-bindgen-futures","web-sys",- "wgpu-core 30.0.1",- "wgpu-hal 30.0.1",- "wgpu-types 30.0.1",+ "wgpu-core",+ "wgpu-hal",+ "wgpu-types",][[package]]@@ -6663,7 +6613,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7"dependencies = ["arrayvec",- "bit-set 0.9.1",+ "bit-set","bit-vec","bitflags 2.13.1","bytemuck",@@ -6672,7 +6622,7 @@ dependencies = ["hashbrown 0.16.1","indexmap","log",- "naga 29.0.4",+ "naga","once_cell","parking_lot","portable-atomic",@@ -6681,46 +6631,12 @@ dependencies = ["rustc-hash 1.1.0","smallvec","thiserror 2.0.20",- "wgpu-core-deps-apple 29.0.4",- "wgpu-core-deps-emscripten 29.0.4",- "wgpu-core-deps-windows-linux-android 29.0.4",- "wgpu-hal 29.0.4",- "wgpu-naga-bridge 29.0.4",- "wgpu-types 29.0.4",-]--[[package]]-name = "wgpu-core"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "14c018fce9b6270aa203c2fdd56f3cce996713534bd757e4ea58c8560b121f14"-dependencies = [- "arrayvec",- "bit-set 0.10.0",- "bit-vec",- "bitflags 2.13.1",- "bytemuck",- "cfg_aliases",- "document-features",- "hashbrown 0.17.1",- "indexmap",- "log",- "naga 30.0.1",- "naga-types",- "once_cell",- "parking_lot",- "portable-atomic",- "profiling",- "raw-window-handle",- "rustc-hash 1.1.0",- "smallvec",- "thiserror 2.0.20",- "wgpu-core-deps-apple 30.0.1",- "wgpu-core-deps-emscripten 30.0.1",- "wgpu-core-deps-windows-linux-android 30.0.1",- "wgpu-hal 30.0.1",- "wgpu-naga-bridge 30.0.1",- "wgpu-types 30.0.1",+ "wgpu-core-deps-apple",+ "wgpu-core-deps-emscripten",+ "wgpu-core-deps-windows-linux-android",+ "wgpu-hal",+ "wgpu-naga-bridge",+ "wgpu-types",][[package]]@@ -6729,16 +6645,7 @@ version = "29.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f5e39e26c4c0e07589e67d18546cf79ff45383659fc72fca4dd293358a0347f3"dependencies = [- "wgpu-hal 29.0.4",-]--[[package]]-name = "wgpu-core-deps-apple"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "061f3d319a40d39d00b1ecc2c33b89fe21d4e6fe01859df3500a3a8ecccd6b68"-dependencies = [- "wgpu-hal 30.0.1",+ "wgpu-hal",][[package]]@@ -6747,16 +6654,7 @@ version = "29.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "01e09be551dc939498bdd5f6b2c66e55ab275dad25825267a08605a80fc9f0af"dependencies = [- "wgpu-hal 29.0.4",-]--[[package]]-name = "wgpu-core-deps-emscripten"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d98b86cf4abf524a902dd35f18ca6a3f08fc2ae9847c8f10b48e30491b1f0b86"-dependencies = [- "wgpu-hal 30.0.1",+ "wgpu-hal",][[package]]@@ -6765,16 +6663,7 @@ version = "29.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4e592c1bbef6ad047647ae6e666ebd8cee7a32bb4544d9700ec96cbf73230257"dependencies = [- "wgpu-hal 29.0.4",-]--[[package]]-name = "wgpu-core-deps-windows-linux-android"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7586165fd5f6d881cb9ce4bb71f40d6caab2c0f1837e3fc1d9788a197fb6004f"-dependencies = [- "wgpu-hal 30.0.1",+ "wgpu-hal",][[package]]@@ -6786,7 +6675,7 @@ dependencies = ["android_system_properties","arrayvec","ash",- "bit-set 0.9.1",+ "bit-set","bitflags 2.13.1","block2 0.6.2","bytemuck",@@ -6802,65 +6691,10 @@ dependencies = ["libc","libloading","log",- "naga 29.0.4",- "ndk-sys",- "objc2 0.6.4",- "objc2-core-foundation",- "objc2-foundation 0.3.2",- "objc2-metal 0.3.2",- "objc2-quartz-core 0.3.2",- "once_cell",- "ordered-float",- "parking_lot",- "portable-atomic",- "portable-atomic-util",- "profiling",- "range-alloc",- "raw-window-handle",- "raw-window-metal",- "renderdoc-sys",- "smallvec",- "thiserror 2.0.20",- "wasm-bindgen",- "wayland-sys",- "web-sys",- "wgpu-naga-bridge 29.0.4",- "wgpu-types 29.0.4",- "windows 0.62.2",- "windows-core 0.62.2",- "windows-result 0.4.1",-]--[[package]]-name = "wgpu-hal"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b6b7fb58561a792bc237628ba0792e332de418fefe145f13b5ed8201e6d52f58"-dependencies = [- "android_system_properties",- "arrayvec",- "ash",- "bit-set 0.10.0",- "bitflags 2.13.1",- "block2 0.6.2",- "bytemuck",- "cfg-if",- "cfg_aliases",- "glow",- "glutin_wgl_sys",- "gpu-allocator",- "hashbrown 0.17.1",- "js-sys",- "khronos-egl",- "libc",- "libloading",- "log",- "naga 30.0.1",- "naga-types",+ "naga","ndk-sys","objc2 0.6.4","objc2-core-foundation",- "objc2-core-graphics","objc2-foundation 0.3.2","objc2-metal 0.3.2","objc2-quartz-core 0.3.2",@@ -6875,13 +6709,12 @@ dependencies = ["raw-window-metal","renderdoc-sys","smallvec",- "static_assertions","thiserror 2.0.20","wasm-bindgen","wayland-sys","web-sys",- "wgpu-naga-bridge 30.0.1",- "wgpu-types 30.0.1",+ "wgpu-naga-bridge",+ "wgpu-types","windows 0.62.2","windows-core 0.62.2","windows-result 0.4.1",@@ -6893,18 +6726,8 @@ version = "29.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "95226013f547544b223281cd16a4fb549aa9dcb562adbda0faae4c73ffbbc161"dependencies = [- "naga 29.0.4",- "wgpu-types 29.0.4",-]--[[package]]-name = "wgpu-naga-bridge"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d2f62e73117bb7a62bfd9c5a5841438a823f6566c6442a808ee269d2d055c081"-dependencies = [- "naga 30.0.1",- "wgpu-types 30.0.1",+ "naga",+ "wgpu-types",][[package]]@@ -6921,22 +6744,6 @@ dependencies = ["web-sys",]-[[package]]-name = "wgpu-types"-version = "30.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "99dad6f1fbdbbdb4c278a6508b059d44688f5cebddf78d005a46a31340269286"-dependencies = [- "bitflags 2.13.1",- "bytemuck",- "js-sys",- "log",- "naga-types",- "raw-window-handle",- "static_assertions",- "web-sys",-]-[[package]]name = "whisper-rs"version = "0.16.0"diff --git a/engine/Cargo.toml b/engine/Cargo.tomlindex f3718fe2..10be1346 100644--- a/engine/Cargo.toml+++ b/engine/Cargo.toml@@ -23,8 +23,11 @@ thiserror = "2"clap = { version = "4", features = ["derive"] }# The GPU compositor, behind concat-render's `gpu` feature. A heavy tree, which-# is exactly why it is optional: engine-only work never pays for it.-wgpu = "30"+# is exactly why it is optional: engine-only work never pays for it. Pinned to+# the major Slint 1.17 can share a device with, so the window's renderer and+# the engine's compositor draw on one device and a preview frame never leaves+# the GPU.+wgpu = "29"pollster = "0.4"[workspace.lints.rust]diff --git a/engine/crates/concat-export/src/lib.rs b/engine/crates/concat-export/src/lib.rsindex 974fc83c..5112fc7f 100644--- a/engine/crates/concat-export/src/lib.rs+++ b/engine/crates/concat-export/src/lib.rs@@ -742,6 +742,53 @@ pub fn preview_frame(pool: &mut concat_media::ReaderPool,request: &PreviewFrameRequest,) -> Result<Vec<u8>, String> {+ let sources = preview_sources(pool, request)?;+ let layers = sources.layers();+ // CPU on purpose: one frame at preview size is milliseconds, and holding+ // a GPU context alive for occasional scrubs is not worth its memory. The+ // window composites on its own device through `preview_sources`.+ let composed = CpuCompositor.composite(request.width, request.height, &layers);+ Ok(composed.into_pixels())+}++/// The decoded pictures under the playhead and where each goes: what a+/// compositor needs to draw the paused monitor's frame, decoded but not yet+/// drawn, so a caller with a GPU can draw them where they are shown.+pub struct PreviewSources {+ sources: Vec<(std::sync::Arc<Frame>, f32, Transform)>,+ width: u32,+ height: u32,+}++impl PreviewSources {+ /// The layers, placed the exporter's way, bottom-most first.+ pub fn layers(&self) -> Vec<Layer<'_>> {+ self.sources+ .iter()+ .map(|(frame, opacity, transform)| {+ place_layer(frame.as_ref(), *opacity, transform, self.width, self.height)+ })+ .collect()+ }++ /// The output width the layers were placed for.+ pub fn width(&self) -> u32 {+ self.width+ }++ /// The output height the layers were placed for.+ pub fn height(&self) -> u32 {+ self.height+ }+}++/// Decodes the pictures the paused monitor's frame is made of, through the+/// reader pool so scrubbing revisits are cache hits. See [`preview_frame`]+/// for the rules on what counts as a failure.+pub fn preview_sources(+ pool: &mut concat_media::ReaderPool,+ request: &PreviewFrameRequest,+) -> Result<PreviewSources, String> {let rate = FrameRate::new(Rational::new(request.rate_num, request.rate_den));let BuiltTimeline {timeline,@@ -786,24 +833,11 @@ pub fn preview_frame());}- // The exporter's own placement, by construction: same function.- let layers: Vec<Layer<'_>> = sources- .iter()- .map(|(frame, opacity, transform)| {- place_layer(- frame.as_ref(),- *opacity,- transform,- request.width,- request.height,- )- })- .collect();-- // CPU on purpose: one frame at preview size is milliseconds, and holding- // a GPU context alive for occasional scrubs is not worth its memory.- let composed = CpuCompositor.composite(request.width, request.height, &layers);- Ok(composed.into_pixels())+ Ok(PreviewSources {+ sources,+ width: request.width,+ height: request.height,+ })}/// The preview's timeline, built the exporter's way.diff --git a/engine/crates/concat-host/Cargo.toml b/engine/crates/concat-host/Cargo.tomlindex e9efea20..50f03d35 100644--- a/engine/crates/concat-host/Cargo.toml+++ b/engine/crates/concat-host/Cargo.toml@@ -9,13 +9,14 @@ description = "What the editor window needs that is not the edit itself: session[features]# The wgpu compositor for previews and exports, off by default for the same# reason as everywhere else in the engine; the window turns it on.-gpu = ["concat-export/gpu"]+gpu = ["concat-export/gpu", "dep:concat-render", "concat-render/gpu"][dependencies]concat-core.workspace = trueconcat-media.workspace = trueconcat-project.workspace = trueconcat-export.workspace = true+concat-render = { workspace = true, optional = true }serde = { version = "1", features = ["derive"] }serde_json.workspace = truediff --git a/engine/crates/concat-host/src/preview.rs b/engine/crates/concat-host/src/preview.rsindex bd0fb007..3093a132 100644--- a/engine/crates/concat-host/src/preview.rs+++ b/engine/crates/concat-host/src/preview.rs@@ -8,6 +8,11 @@//! at a time, in order, on whatever thread the caller chose - the window//! debounces and drops stale results, so a slow decode never wedges anything//! but itself.+//!+//! With the `gpu` feature and a device from [`Monitor::with_gpu`], a frame+//! is composited on that device and handed back as a texture: decoded+//! pictures go up once, the composite happens where it is shown, and no+//! pixel comes back down.use std::sync::{Arc, Mutex};@@ -30,8 +35,14 @@ pub struct FrameSpec {#[derive(Clone)]pub struct Monitor {pool: Arc<Mutex<concat_media::ReaderPool>>,+ #[cfg(feature = "gpu")]+ gpu: Option<Arc<Mutex<concat_render::WgpuCompositor>>>,}+/// The wgpu the monitor's textures belong to.+#[cfg(feature = "gpu")]+pub use concat_render::wgpu;+impl Default for Monitor {fn default() -> Self {Self::new()@@ -43,9 +54,63 @@ impl Monitor {pub fn new() -> Self {Self {pool: Arc::new(Mutex::new(concat_media::ReaderPool::with_defaults())),+ #[cfg(feature = "gpu")]+ gpu: None,+ }+ }++ /// A monitor that composites on `device` - the window's - so+ /// [`Monitor::frame_texture`] yields textures the window shows as they+ /// are.+ #[cfg(feature = "gpu")]+ pub fn with_gpu(device: wgpu::Device, queue: wgpu::Queue) -> Self {+ Self {+ pool: Arc::new(Mutex::new(concat_media::ReaderPool::with_defaults())),+ gpu: Some(Arc::new(Mutex::new(+ concat_render::WgpuCompositor::with_device(device, queue),+ ))),+ }+ }++ /// Whether frames can be composited on the GPU.+ pub fn has_gpu(&self) -> bool {+ #[cfg(feature = "gpu")]+ {+ self.gpu.is_some()+ }+ #[cfg(not(feature = "gpu"))]+ {+ false}}+ /// The engine-composited frame at one instant, as a texture on the+ /// device this monitor was given: `Rgba8Unorm`, `spec.width` by+ /// `spec.height`, bindable and renderable. Errs without a device, or+ /// once the device is lost.+ #[cfg(feature = "gpu")]+ pub fn frame_texture(+ &self,+ clips: Vec<ExportClip>,+ settings: &DocumentSettings,+ spec: FrameSpec,+ ) -> Result<wgpu::Texture, String> {+ let gpu = self+ .gpu+ .as_ref()+ .ok_or_else(|| "the monitor has no GPU device".to_owned())?;+ let request = Self::request(clips, settings, spec);+ let mut pool = self+ .pool+ .lock()+ .map_err(|_| "reader pool poisoned".to_owned())?;+ let sources = concat_export::preview_sources(&mut pool, &request)?;+ let layers = sources.layers();+ let mut gpu = gpu.lock().map_err(|_| "compositor poisoned".to_owned())?;+ gpu.composite_texture(spec.width, spec.height, &layers)+ .ok_or_else(|| "the GPU device was lost".to_owned())+ }+fn request(clips: Vec<ExportClip>,settings: &DocumentSettings,diff --git a/engine/crates/concat-render/src/gpu.rs b/engine/crates/concat-render/src/gpu.rsindex 2a15096f..33a55cb6 100644--- a/engine/crates/concat-render/src/gpu.rs+++ b/engine/crates/concat-render/src/gpu.rs@@ -19,6 +19,12 @@//!//! Construction is fallible: a machine with no usable adapter gets `None`, and//! callers fall back to the CPU. Never panic over a missing GPU.+//!+//! Two outputs. [`Compositor::composite`] reads the frame back for the+//! encoder. [`WgpuCompositor::composite_texture`] leaves it on the GPU as a+//! texture the window can show directly - when the compositor was built on+//! the window's own device with [`WgpuCompositor::with_device`], that is the+//! monitor with no copy anywhere.use std::collections::HashMap;@@ -103,6 +109,19 @@ struct Target {padded_row: usize,}+/// Presentable output textures, in a ring: the window may still be+/// sampling the last one while the next is drawn.+struct Presentable {+ width: u32,+ height: u32,+ ring: Vec<wgpu::Texture>,+ next: usize,+}++/// How many presentable textures are kept: the one on screen, the one being+/// drawn, and one so a late frame never waits on either.+const PRESENT_RING: usize = 3;+/// A compositor that draws on the GPU. See the module docs.pub struct WgpuCompositor {device: wgpu::Device,@@ -120,6 +139,7 @@ pub struct WgpuCompositor {used: HashMap<(u32, u32), usize>,idle: HashMap<(u32, u32), u32>,target: Option<Target>,+ presentable: Option<Presentable>,/// Set when a readback fails - a lost or reset device. The compositor/// then answers every composite from the CPU reference instead: slower,/// always correct, and never a panic in the middle of an export.@@ -138,7 +158,12 @@ impl WgpuCompositor {.ok()?;let (device, queue) =pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()?;+ Some(Self::with_device(device, queue))+ }+ /// Builds a compositor on a device the caller owns - the window's, so a+ /// texture this draws is one the window can show.+ pub fn with_device(device: wgpu::Device, queue: wgpu::Queue) -> Self {let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {label: Some("concat compositor"),source: wgpu::ShaderSource::Wgsl(SHADER.into()),@@ -201,7 +226,7 @@ impl WgpuCompositor {module: &shader,entry_point: Some("vs_main"),compilation_options: Default::default(),- buffers: &[Some(vertex_layout)],+ buffers: &[vertex_layout],},fragment: Some(wgpu::FragmentState {module: &shader,@@ -236,7 +261,7 @@ impl WgpuCompositor {mapped_at_creation: false,});- Some(Self {+ Self {device,queue,pipeline,@@ -247,9 +272,173 @@ impl WgpuCompositor {pool: HashMap::new(),used: HashMap::new(),target: None,+ presentable: None,idle: HashMap::new(),dead: false,- })+ }+ }++ /// Whether the device has been lost. A dead compositor answers+ /// [`Compositor::composite`] from the CPU and refuses textures.+ pub fn is_dead(&self) -> bool {+ self.dead+ }++ /// The next presentable texture for this output size.+ fn presentable(&mut self, width: u32, height: u32) -> wgpu::Texture {+ let stale = self+ .presentable+ .as_ref()+ .is_none_or(|p| p.width != width || p.height != height);+ if stale {+ let ring = (0..PRESENT_RING)+ .map(|_| {+ self.device.create_texture(&wgpu::TextureDescriptor {+ label: Some("concat monitor"),+ size: wgpu::Extent3d {+ width,+ height,+ depth_or_array_layers: 1,+ },+ mip_level_count: 1,+ sample_count: 1,+ dimension: wgpu::TextureDimension::D2,+ format: wgpu::TextureFormat::Rgba8Unorm,+ usage: wgpu::TextureUsages::RENDER_ATTACHMENT+ | wgpu::TextureUsages::TEXTURE_BINDING+ | wgpu::TextureUsages::COPY_SRC,+ view_formats: &[],+ })+ })+ .collect();+ self.presentable = Some(Presentable {+ width,+ height,+ ring,+ next: 0,+ });+ }+ let presentable = self.presentable.as_mut().expect("just ensured");+ let texture = presentable.ring[presentable.next].clone();+ presentable.next = (presentable.next + 1) % PRESENT_RING;+ texture+ }++ /// Uploads every visible layer and writes its quad; the draws, in order.+ fn prepare(&mut self, width: u32, height: u32, layers: &[Layer<'_>]) -> Vec<(u32, u32, usize)> {+ self.used.values_mut().for_each(|used| *used = 0);++ let mut draws: Vec<(u32, u32, usize)> = Vec::with_capacity(layers.len());+ let mut vertices: Vec<Vertex> = Vec::with_capacity(layers.len() * 6);+ for layer in layers {+ if layer.opacity <= 0.0 {+ continue;+ }+ let index = self.upload(layer.frame);+ draws.push((layer.frame.width(), layer.frame.height(), index));+ vertices.extend_from_slice(&Self::quad(layer, width, height));+ }++ if vertices.len() > self.vertex_capacity {+ self.vertex_capacity = vertices.len().next_power_of_two();+ self.vertices = self.device.create_buffer(&wgpu::BufferDescriptor {+ label: Some("concat quads"),+ size: (std::mem::size_of::<Vertex>() * self.vertex_capacity) as u64,+ usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST,+ mapped_at_creation: false,+ });+ }+ if !vertices.is_empty() {+ self.queue+ .write_buffer(&self.vertices, 0, as_bytes(&vertices));+ }+ draws+ }++ /// The render pass: every draw over black into `view`. Returns the+ /// encoder so the caller can add a readback before submitting.+ fn encode(+ &self,+ view: &wgpu::TextureView,+ draws: &[(u32, u32, usize)],+ ) -> wgpu::CommandEncoder {+ let mut encoder = self+ .device+ .create_command_encoder(&wgpu::CommandEncoderDescriptor {+ label: Some("concat composite"),+ });+ {+ let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {+ label: Some("concat composite"),+ color_attachments: &[Some(wgpu::RenderPassColorAttachment {+ view,+ depth_slice: None,+ resolve_target: None,+ ops: wgpu::Operations {+ load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),+ store: wgpu::StoreOp::Store,+ },+ })],+ depth_stencil_attachment: None,+ timestamp_writes: None,+ occlusion_query_set: None,+ multiview_mask: None,+ });+ pass.set_pipeline(&self.pipeline);+ pass.set_vertex_buffer(0, self.vertices.slice(..));+ for (draw, (layer_width, layer_height, pooled)) in draws.iter().enumerate() {+ let bind_group = &self.pool[&(*layer_width, *layer_height)][*pooled].bind_group;+ pass.set_bind_group(0, bind_group, &[]);+ let first = (draw * 6) as u32;+ pass.draw(first..first + 6, 0..1);+ }+ }+ encoder+ }++ /// Retires texture sizes the timeline has moved past. 300 unclaimed+ /// composites (ten seconds of 30fps export) says a size is gone for+ /// good, not just between two clips of it.+ fn retire(&mut self) {+ for (&key, used) in &self.used {+ let idle = self.idle.entry(key).or_insert(0);+ *idle = if *used == 0 { *idle + 1 } else { 0 };+ }+ let doomed: Vec<(u32, u32)> = self+ .idle+ .iter()+ .filter(|(_, idle)| **idle > 300)+ .map(|(key, _)| *key)+ .collect();+ for key in doomed {+ self.pool.remove(&key);+ self.used.remove(&key);+ self.idle.remove(&key);+ }+ }++ /// Composites `layers` into a texture that stays on the GPU, and hands+ /// it back: `Rgba8Unorm`, bindable and renderable, exactly `width` by+ /// `height`. The texture is one of a small ring, so the caller may keep+ /// showing the previous one while this draws. `None` when the device is+ /// dead; the caller then falls back to [`Compositor::composite`] on a+ /// CPU compositor.+ pub fn composite_texture(+ &mut self,+ width: u32,+ height: u32,+ layers: &[Layer<'_>],+ ) -> Option<wgpu::Texture> {+ if self.dead {+ return None;+ }+ let draws = self.prepare(width, height, layers);+ let texture = self.presentable(width, height);+ let view = texture.create_view(&wgpu::TextureViewDescriptor::default());+ let encoder = self.encode(&view, &draws);+ self.queue.submit([encoder.finish()]);+ self.retire();+ Some(texture)}/// The reusable render target for this output size.@@ -419,7 +608,7 @@ impl WgpuCompositor {let mut frame = Frame::transparent(width, height);{- let data = slice.get_mapped_range().ok()?;+ let data = slice.get_mapped_range();let row_bytes = width as usize * 4;let pixels = frame.pixels_mut();for row in 0..height as usize {@@ -445,72 +634,13 @@ impl Compositor for WgpuCompositor {return CpuCompositor.composite(width, height, layers);}- self.used.values_mut().for_each(|used| *used = 0);-- // Upload every visible layer and build its quad.- let mut draws: Vec<(u32, u32, usize)> = Vec::with_capacity(layers.len());- let mut vertices: Vec<Vertex> = Vec::with_capacity(layers.len() * 6);- for layer in layers {- if layer.opacity <= 0.0 {- continue;- }- let index = self.upload(layer.frame);- draws.push((layer.frame.width(), layer.frame.height(), index));- vertices.extend_from_slice(&Self::quad(layer, width, height));- }-- if vertices.len() > self.vertex_capacity {- self.vertex_capacity = vertices.len().next_power_of_two();- self.vertices = self.device.create_buffer(&wgpu::BufferDescriptor {- label: Some("concat quads"),- size: (std::mem::size_of::<Vertex>() * self.vertex_capacity) as u64,- usage: wgpu::BufferUsages::VERTEX | wgpu::BufferUsages::COPY_DST,- mapped_at_creation: false,- });- }- if !vertices.is_empty() {- self.queue- .write_buffer(&self.vertices, 0, as_bytes(&vertices));- }-+ let draws = self.prepare(width, height, layers);self.target(width, height);let target = self.target.as_ref().expect("just ensured");let view = target.texture.create_view(&wgpu::TextureViewDescriptor::default());-- let mut encoder = self- .device- .create_command_encoder(&wgpu::CommandEncoderDescriptor {- label: Some("concat composite"),- });- {- let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {- label: Some("concat composite"),- color_attachments: &[Some(wgpu::RenderPassColorAttachment {- view: &view,- depth_slice: None,- resolve_target: None,- ops: wgpu::Operations {- load: wgpu::LoadOp::Clear(wgpu::Color::BLACK),- store: wgpu::StoreOp::Store,- },- })],- depth_stencil_attachment: None,- timestamp_writes: None,- occlusion_query_set: None,- multiview_mask: None,- });- pass.set_pipeline(&self.pipeline);- pass.set_vertex_buffer(0, self.vertices.slice(..));- for (draw, (layer_width, layer_height, pooled)) in draws.iter().enumerate() {- let bind_group = &self.pool[&(*layer_width, *layer_height)][*pooled].bind_group;- pass.set_bind_group(0, bind_group, &[]);- let first = (draw * 6) as u32;- pass.draw(first..first + 6, 0..1);- }- }-+ let mut encoder = self.encode(&view, &draws);encoder.copy_texture_to_buffer(wgpu::TexelCopyTextureInfo {texture: &target.texture,@@ -532,27 +662,8 @@ impl Compositor for WgpuCompositor {depth_or_array_layers: 1,},);-self.queue.submit([encoder.finish()]);-- // Retire texture sizes the timeline has moved past. 300 unclaimed- // composites (ten seconds of 30fps export) says a size is gone for- // good, not just between two clips of it.- for (&key, used) in &self.used {- let idle = self.idle.entry(key).or_insert(0);- *idle = if *used == 0 { *idle + 1 } else { 0 };- }- let doomed: Vec<(u32, u32)> = self- .idle- .iter()- .filter(|(_, idle)| **idle > 300)- .map(|(key, _)| *key)- .collect();- for key in doomed {- self.pool.remove(&key);- self.used.remove(&key);- self.idle.remove(&key);- }+ self.retire();match self.read_back() {Some(frame) => frame,diff --git a/engine/crates/concat-render/src/lib.rs b/engine/crates/concat-render/src/lib.rsindex d0a46d31..e7da632c 100644--- a/engine/crates/concat-render/src/lib.rs+++ b/engine/crates/concat-render/src/lib.rs@@ -22,3 +22,6 @@ pub use compositor::{Compositor, CpuCompositor, Layer, Placement};#[cfg(feature = "gpu")]pub use gpu::WgpuCompositor;pub use plan::{FramePlan, PlannedLayer, plan_frame};+/// The wgpu the compositor is built on, for callers that share its device.+#[cfg(feature = "gpu")]+pub use wgpu;diff --git a/engine/crates/concat/Cargo.toml b/engine/crates/concat/Cargo.tomlindex d9e27ff0..13a7eba7 100644--- a/engine/crates/concat/Cargo.toml+++ b/engine/crates/concat/Cargo.toml@@ -36,8 +36,16 @@ slint = { version = "1.17", default-features = false, features = [# title bar (macOS traffic-light overlay) and to start window drags from# the strip's drag region."unstable-winit-030",+ # The renderer draws on a wgpu device the window creates and shares with+ # the engine's compositor, so a monitor frame is a texture Slint shows+ # as it is - see gpu.rs.+ "unstable-wgpu-29",] }+# The one device: created in gpu.rs, handed to Slint and to the monitor.+wgpu.workspace = true+pollster.workspace = true+# The engine, through the host layer. `gpu` turns on the wgpu compositor for# the monitor and the export; the CPU compositor stays as the fallback.concat-core.workspace = truediff --git a/engine/crates/concat/src/gpu.rs b/engine/crates/concat/src/gpu.rsnew file mode 100644index 00000000..7aa09120--- /dev/null+++ b/engine/crates/concat/src/gpu.rs@@ -0,0 +1,60 @@+// SPDX-License-Identifier: AGPL-3.0-or-later+// SPDX-FileCopyrightText: 2026 Jareer and Concat contributors++//! The one GPU device the window and the engine share.+//!+//! Slint renders the window on it, and the monitor composites on it. Because+//! both sides hold the same device, a composited frame is a texture the+//! renderer samples directly: nothing is read back and nothing is copied.+//! Created before the backend is selected, because Slint takes the device at+//! selection time; `None` when the machine offers no adapter, and then the+//! window renders the way it did without one.++/// The shared device and what it was created from.+#[derive(Clone)]+pub struct Gpu {+ /// The instance, which Slint needs to make a surface for the window.+ pub instance: wgpu::Instance,+ /// The adapter, which Slint reads the backend off.+ pub adapter: wgpu::Adapter,+ /// The device both sides draw on.+ pub device: wgpu::Device,+ /// The one queue, so submissions from both sides are ordered.+ pub queue: wgpu::Queue,+}++impl Gpu {+ /// Opens the machine's best adapter and a device on it.+ pub fn acquire() -> Option<Gpu> {+ let instance = wgpu::Instance::new(wgpu::InstanceDescriptor {+ backends: wgpu::Backends::PRIMARY,+ ..wgpu::InstanceDescriptor::new_without_display_handle()+ });+ let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions {+ power_preference: wgpu::PowerPreference::HighPerformance,+ ..Default::default()+ }))+ .ok()?;+ let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor {+ label: Some("concat"),+ ..Default::default()+ }))+ .ok()?;+ Some(Gpu {+ instance,+ adapter,+ device,+ queue,+ })+ }++ /// The device as Slint takes it.+ pub fn configuration(&self) -> slint::wgpu_29::WGPUConfiguration {+ slint::wgpu_29::WGPUConfiguration::Manual {+ instance: self.instance.clone(),+ adapter: self.adapter.clone(),+ device: self.device.clone(),+ queue: self.queue.clone(),+ }+ }+}diff --git a/engine/crates/concat/src/host.rs b/engine/crates/concat/src/host.rsindex f9922c6e..11e4b72c 100644--- a/engine/crates/concat/src/host.rs+++ b/engine/crates/concat/src/host.rs@@ -21,6 +21,7 @@ use concat_host::preview::Monitor;use concat_host::{AppDirs, media};use concat_speech::{Speech, Transcriber};+use crate::gpu::Gpu;use crate::studio::{Models, Studio};use crate::ui::App;@@ -42,14 +43,19 @@ pub struct Host {impl Host {/// Starts every service. The audio device may not be there yet; playback- /// keeps trying on its own thread and says so through a toast.- pub fn start() -> Result<Host, String> {+ /// keeps trying on its own thread and says so through a toast. `gpu` is+ /// the window's device; with it the monitor composites where the window+ /// draws.+ pub fn start(gpu: Option<Gpu>) -> Result<Host, String> {let dirs = AppDirs::locate()?;let _ = std::fs::create_dir_all(&dirs.config);Ok(Host {dirs,playback: Playback::start(Arc::new(Events)),- monitor: Monitor::new(),+ monitor: match gpu {+ Some(gpu) => Monitor::with_gpu(gpu.device, gpu.queue),+ None => Monitor::new(),+ },exporter: Exporter::new(),transcriber: Arc::new(Transcriber::new()),speech: Arc::new(Speech::new()),diff --git a/engine/crates/concat/src/main.rs b/engine/crates/concat/src/main.rsindex fbf8117b..5661b89f 100644--- a/engine/crates/concat/src/main.rs+++ b/engine/crates/concat/src/main.rs@@ -33,6 +33,7 @@ mod ui {mod chips;mod dock;mod format;+mod gpu;mod host;mod prefs;mod studio;@@ -44,38 +45,52 @@ use studio::{LANGUAGES, Models, OUTPUTS, RESOLUTIONS, START_RATES, Studio};use ui::*;fn main() -> Result<(), slint::PlatformError> {+ // The device the renderer and the monitor share. Taken first, because+ // the backend is selected with it.+ let gpu = gpu::Gpu::acquire();+ if gpu.is_none() {+ eprintln!("concat: no GPU adapter; the monitor composites on the CPU");+ }++ let mut selector = slint::BackendSelector::new().backend_name("winit".into());+ selector = match &gpu {+ Some(gpu) => selector.require_wgpu_29(gpu.configuration()),+ None => {+ // Without a shared device, ask for the platform's own API by+ // name: Skia picks its surface from a cfg chain, and requiring+ // one turns a silent fall back to the CPU rasteriser into a+ // refusal to start, which is a fault you can see.+ #[cfg(target_os = "macos")]+ {+ selector.require_metal()+ }+ #[cfg(target_family = "windows")]+ {+ selector.require_d3d()+ }+ #[cfg(not(any(target_os = "macos", target_family = "windows")))]+ {+ selector+ }+ }+ };+// The custom title bar. On macOS the native bar is hidden and the traffic// lights are overlaid on the strip the UI draws. Other platforms keep// their decorations for now.#[cfg(target_os = "macos")]{use slint::winit_030::winit::platform::macos::WindowAttributesExtMacOS;- slint::BackendSelector::new()- .backend_name("winit".into())- // Metal, by name. Skia picks its surface from a cfg chain that on- // macOS already lands on Metal; requiring it turns a silent fall- // back to the CPU rasteriser into a refusal to start, which is a- // fault you can see.- .require_metal()- .with_winit_window_attributes_hook(|attributes| {- attributes- .with_titlebar_transparent(true)- .with_title_hidden(true)- .with_fullsize_content_view(true)- })- .select()?;+ selector = selector.with_winit_window_attributes_hook(|attributes| {+ attributes+ .with_titlebar_transparent(true)+ .with_title_hidden(true)+ .with_fullsize_content_view(true)+ });}+ selector.select()?;- // Windows renders on the CPU unless Direct3D is asked for by name: Skia's- // surface chain has no Direct3D arm and its OpenGL arm excludes Windows,- // so plain renderer-skia lands on the software surface there.- #[cfg(target_family = "windows")]- slint::BackendSelector::new()- .backend_name("winit".into())- .require_d3d()- .select()?;-- let host = match Host::start() {+ let host = match Host::start(gpu) {Ok(host) => host,Err(error) => {eprintln!("concat: {error}");diff --git a/engine/crates/concat/src/studio.rs b/engine/crates/concat/src/studio.rsindex 72cf4374..b535c17d 100644--- a/engine/crates/concat/src/studio.rs+++ b/engine/crates/concat/src/studio.rs@@ -835,6 +835,14 @@ impl Studio {/// Asks the engine for the frame at the playhead, one at a time: a/// request while one is out waits for it, and the newest wins.pub fn request_preview(&mut self) {+ /// A monitor frame on its way to the window.+ enum Picture {+ /// Already on the GPU, on the window's own device.+ Texture(concat_host::preview::wgpu::Texture),+ /// Raw RGBA, to be uploaded.+ Pixels(Vec<u8>, u32, u32),+ }+if self.on_start || self.session.is_none() {return;}@@ -865,15 +873,29 @@ impl Studio {self.preview_wanted = false;spawn(move || {- let frame = monitor.frame(clips.clone(), &settings, spec);+ // On the window's device the frame stays a texture; without+ // one it comes back as pixels and is uploaded here.+ let frame = if monitor.has_gpu() {+ monitor+ .frame_texture(clips.clone(), &settings, spec)+ .map(Picture::Texture)+ } else {+ monitor+ .frame(clips.clone(), &settings, spec)+ .map(|bytes| Picture::Pixels(bytes, width, height))+ };// Decode-ahead for whatever comes next, while the pool is warm.monitor.prefetch(clips, &settings, spec, 2);- frame.map(|bytes| (bytes, width, height))+ frame},|studio, _, _, result| {studio.preview_busy = false;match result {- Ok((bytes, width, height)) => {+ Ok(Picture::Texture(texture)) => match slint::Image::try_from(texture) {+ Ok(image) => studio.preview = image,+ Err(error) => eprintln!("concat: preview texture: {error}"),+ },+ Ok(Picture::Pixels(bytes, width, height)) => {let buffer =slint::SharedPixelBuffer::<slint::Rgba8Pixel>::clone_from_slice(&bytes, width, height,