Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(platform-linux): native-Wayland parity (wlroots + portal + libei) (#1966)
Francesco Bonacci519dfdfbd6c7
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lockindex 362556d4dd..c0805d8e90 100644--- a/libs/cua-driver/rust/Cargo.lock+++ b/libs/cua-driver/rust/Cargo.lock@@ -17,6 +17,16 @@ dependencies = ["memchr",]+[[package]]+name = "annotate-snippets"+version = "0.9.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ccaf7e9dfbb6ab22c82e473cd1a8a7bd313c19a5b7e40970f3d89ef5a5c9e81e"+dependencies = [+ "unicode-width",+ "yansi-term",+]+[[package]]name = "anyhow"version = "1.0.102"@@ -53,6 +63,21 @@ version = "0.7.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"+[[package]]+name = "ashpd"+version = "0.13.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "340e0f6bf7f9ee78549c61454f1460a3ed97c011902ee76b58301bbc6d502a32"+dependencies = [+ "enumflags2",+ "futures-util",+ "getrandom 0.4.2",+ "serde",+ "serde_repr",+ "tokio",+ "zbus",+]+[[package]]name = "async-broadcast"version = "0.7.2"@@ -253,6 +278,27 @@ version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"+[[package]]+name = "bindgen"+version = "0.69.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"+dependencies = [+ "annotate-snippets",+ "bitflags 2.11.1",+ "cexpr",+ "clang-sys",+ "itertools",+ "lazy_static",+ "lazycell",+ "proc-macro2",+ "quote",+ "regex",+ "rustc-hash",+ "shlex",+ "syn",+]+[[package]]name = "bitflags"version = "1.3.2"@@ -338,6 +384,19 @@ version = "1.11.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"+[[package]]+name = "calloop"+version = "0.14.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7"+dependencies = [+ "bitflags 2.11.1",+ "polling",+ "rustix",+ "slab",+ "tracing",+]+[[package]]name = "cc"version = "1.2.62"@@ -348,12 +407,42 @@ dependencies = ["shlex",]+[[package]]+name = "cexpr"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"+dependencies = [+ "nom 7.1.3",+]++[[package]]+name = "cfg-expr"+version = "0.15.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02"+dependencies = [+ "smallvec",+ "target-lexicon",+]+[[package]]name = "cfg-if"version = "1.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"+[[package]]+name = "clang-sys"+version = "1.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"+dependencies = [+ "glob",+ "libc",+ "libloading",+]+[[package]]name = "concurrent-queue"version = "2.5.0"@@ -363,6 +452,15 @@ dependencies = ["crossbeam-utils",]+[[package]]+name = "convert_case"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"+dependencies = [+ "unicode-segmentation",+]+[[package]]name = "cookie"version = "0.18.1"@@ -374,6 +472,15 @@ dependencies = ["version_check",]+[[package]]+name = "cookie-factory"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2"+dependencies = [+ "futures",+]+[[package]]name = "cookie_store"version = "0.22.1"@@ -459,6 +566,15 @@ dependencies = ["cfg-if",]+[[package]]+name = "crossbeam-channel"+version = "0.5.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"+dependencies = [+ "crossbeam-utils",+]+[[package]]name = "crossbeam-queue"version = "0.3.12"@@ -590,6 +706,27 @@ dependencies = ["crypto-common",]+[[package]]+name = "dirs"+version = "5.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"+dependencies = [+ "dirs-sys",+]++[[package]]+name = "dirs-sys"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"+dependencies = [+ "libc",+ "option-ext",+ "redox_users",+ "windows-sys 0.48.0",+]+[[package]]name = "displaydoc"version = "0.2.5"@@ -626,6 +763,12 @@ version = "1.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"+[[package]]+name = "either"+version = "1.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"+[[package]]name = "embed-manifest"version = "1.5.0"@@ -707,7 +850,7 @@ dependencies = ["bitvec","cfg-if","libc",- "nix",+ "nix 0.23.2","thiserror",]@@ -867,12 +1010,48 @@ version = "2.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"+[[package]]+name = "futures"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"+dependencies = [+ "futures-channel",+ "futures-core",+ "futures-executor",+ "futures-io",+ "futures-sink",+ "futures-task",+ "futures-util",+]++[[package]]+name = "futures-channel"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"+dependencies = [+ "futures-core",+ "futures-sink",+]+[[package]]name = "futures-core"version = "0.3.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"+[[package]]+name = "futures-executor"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"+dependencies = [+ "futures-core",+ "futures-task",+ "futures-util",+]+[[package]]name = "futures-io"version = "0.3.32"@@ -921,10 +1100,13 @@ version = "0.3.32"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"dependencies = [+ "futures-channel","futures-core",+ "futures-io","futures-macro","futures-sink","futures-task",+ "memchr","pin-project-lite","slab",]@@ -973,6 +1155,12 @@ dependencies = ["wasip3",]+[[package]]+name = "glob"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"+[[package]]name = "hashbrown"version = "0.15.5"@@ -1164,6 +1352,15 @@ dependencies = ["serde_core",]+[[package]]+name = "itertools"+version = "0.12.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"+dependencies = [+ "either",+]+[[package]]name = "itoa"version = "1.0.18"@@ -1199,6 +1396,12 @@ version = "1.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"+[[package]]+name = "lazycell"+version = "1.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"+[[package]]name = "leb128fmt"version = "0.1.0"@@ -1211,12 +1414,59 @@ version = "0.2.186"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"+[[package]]+name = "libloading"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"+dependencies = [+ "cfg-if",+ "windows-link 0.2.1",+]+[[package]]name = "libm"version = "0.2.16"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"+[[package]]+name = "libredox"+version = "0.1.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3"+dependencies = [+ "libc",+]++[[package]]+name = "libspa"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "65f3a4b81b2a2d8c7f300643676202debd1b7c929dbf5c9bb89402ea11d19810"+dependencies = [+ "bitflags 2.11.1",+ "cc",+ "convert_case",+ "cookie-factory",+ "libc",+ "libspa-sys",+ "nix 0.27.1",+ "nom 7.1.3",+ "system-deps",+]++[[package]]+name = "libspa-sys"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bf0d9716420364790e85cbb9d3ac2c950bde16a7dd36f3209b7dfdfc4a24d01f"+dependencies = [+ "bindgen",+ "cc",+ "system-deps",+]+[[package]]name = "linux-raw-sys"version = "0.12.1"@@ -1265,6 +1515,15 @@ version = "2.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"+[[package]]+name = "memmap2"+version = "0.9.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"+dependencies = [+ "libc",+]+[[package]]name = "memoffset"version = "0.6.5"@@ -1290,9 +1549,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f79496a5651c8d57cd033c5add8ca7ee4e3d5f7587a4777484640d9cb60392d9"dependencies = ["fnv",- "nom",+ "nom 1.2.4",]+[[package]]+name = "minimal-lexical"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"+[[package]]name = "miniz_oxide"version = "0.8.9"@@ -1354,12 +1619,33 @@ dependencies = ["memoffset 0.6.5",]+[[package]]+name = "nix"+version = "0.27.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"+dependencies = [+ "bitflags 2.11.1",+ "cfg-if",+ "libc",+]+[[package]]name = "nom"version = "1.2.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a5b8c256fd9471521bcb84c3cdba98921497f1a331cbc15b8030fc63b82050ce"+[[package]]+name = "nom"+version = "7.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"+dependencies = [+ "memchr",+ "minimal-lexical",+]+[[package]]name = "nu-ansi-term"version = "0.50.3"@@ -1532,6 +1818,12 @@ dependencies = ["vcpkg",]+[[package]]+name = "option-ext"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"+[[package]]name = "ordered-stream"version = "0.2.0"@@ -1609,6 +1901,34 @@ dependencies = ["futures-io",]+[[package]]+name = "pipewire"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08e645ba5c45109106d56610b3ee60eb13a6f2beb8b74f8dc8186cf261788dda"+dependencies = [+ "anyhow",+ "bitflags 2.11.1",+ "libc",+ "libspa",+ "libspa-sys",+ "nix 0.27.1",+ "once_cell",+ "pipewire-sys",+ "thiserror",+]++[[package]]+name = "pipewire-sys"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "849e188f90b1dda88fe2bfe1ad31fe5f158af2c98f80fb5d13726c44f3f01112"+dependencies = [+ "bindgen",+ "libspa-sys",+ "system-deps",+]+[[package]]name = "pkg-config"version = "0.3.33"@@ -1620,26 +1940,39 @@ name = "platform-linux"version = "0.6.0"dependencies = ["anyhow",+ "ashpd","async-trait","atspi","base64",+ "calloop",+ "crossbeam-channel","cua-driver-core","cursor-overlay",+ "dirs",+ "enumflags2","evdev","image","libc",+ "libspa","meval","pip-preview",+ "pipewire",+ "reis","serde","serde_json","thiserror","tiny-skia","tokio","tracing",+ "tracing-subscriber",+ "url","wayland-client",+ "wayland-protocols","wayland-protocols-wlr","x11","x11rb",+ "xkbcommon",+ "zbus",][[package]]@@ -1864,6 +2197,29 @@ dependencies = ["bitflags 2.11.1",]+[[package]]+name = "redox_users"+version = "0.4.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"+dependencies = [+ "getrandom 0.2.17",+ "libredox",+ "thiserror",+]++[[package]]+name = "regex"+version = "1.12.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-automata",+ "regex-syntax",+]+[[package]]name = "regex-automata"version = "0.4.14"@@ -1881,6 +2237,19 @@ version = "0.8.10"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"+[[package]]+name = "reis"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "81f3fedd2777cde52c1be5e572efbec485eac7b801c47820eda388d4f13b9c4b"+dependencies = [+ "enumflags2",+ "futures-util",+ "log",+ "rustix",+ "tokio",+]+[[package]]name = "resvg"version = "0.44.0"@@ -1924,6 +2293,12 @@ version = "0.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"+[[package]]+name = "rustc-hash"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"+[[package]]name = "rustc_version"version = "0.4.1"@@ -2269,6 +2644,19 @@ dependencies = ["syn",]+[[package]]+name = "system-deps"+version = "6.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349"+dependencies = [+ "cfg-expr",+ "heck",+ "pkg-config",+ "toml",+ "version-compare",+]+[[package]]name = "tap"version = "1.0.1"@@ -2285,6 +2673,12 @@ dependencies = ["libc",]+[[package]]+name = "target-lexicon"+version = "0.12.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"+[[package]]name = "tempfile"version = "3.27.0"@@ -2539,6 +2933,7 @@ version = "0.1.44"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"dependencies = [+ "log","pin-project-lite","tracing-attributes","tracing-core",@@ -2675,12 +3070,24 @@ version = "0.5.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee"+[[package]]+name = "unicode-segmentation"+version = "1.13.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"+[[package]]name = "unicode-vo"version = "0.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b1d386ff53b415b7fe27b50bb44679e2cc4660272694b7b6f3326d8480823a94"+[[package]]+name = "unicode-width"+version = "0.1.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"+[[package]]name = "unicode-xid"version = "0.2.6"@@ -2806,6 +3213,12 @@ version = "0.2.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"+[[package]]+name = "version-compare"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e"+[[package]]name = "version_check"version = "0.9.5"@@ -3023,6 +3436,28 @@ dependencies = ["rustls-pki-types",]+[[package]]+name = "winapi"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"+dependencies = [+ "winapi-i686-pc-windows-gnu",+ "winapi-x86_64-pc-windows-gnu",+]++[[package]]+name = "winapi-i686-pc-windows-gnu"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"++[[package]]+name = "winapi-x86_64-pc-windows-gnu"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"+[[package]]name = "windows"version = "0.58.0"@@ -3516,12 +3951,38 @@ version = "0.13.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd"+[[package]]+name = "xkbcommon"+version = "0.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a7a974f48060a14e95705c01f24ad9c3345022f4d97441b8a36beb7ed5c4a02d"+dependencies = [+ "libc",+ "memmap2",+ "xkeysym",+]++[[package]]+name = "xkeysym"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56"+[[package]]name = "xmlwriter"version = "0.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9"+[[package]]+name = "yansi-term"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1"+dependencies = [+ "winapi",+]+[[package]]name = "yoke"version = "0.8.2"diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.tomlindex 77515608c1..0138e3fca1 100644--- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml+++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml@@ -38,3 +38,68 @@ meval = "0.2"# when running under a Wayland compositor with no X11 (no DISPLAY).wayland-client = "0.31"wayland-protocols-wlr = { version = "0.3", features = ["client"] }+# Wayland staging protocols: ext-foreign-toplevel-list-v1,+# ext-image-capture-source-v1, ext-image-copy-capture-v1 (per-window+# screenshot on modern wlroots/GNOME/KDE) and wp-fractional-scale-v1 ++# wp-viewporter (HiDPI for the layer-shell overlay). The transitive pull+# from wayland-protocols-wlr does NOT enable `staging`; declaring it here+# unifies the feature globally.+wayland-protocols = { version = "0.32", features = ["client", "staging"] }+# xdg-desktop-portal client: display-screenshot tier+# (org.freedesktop.portal.Screenshot), per-window ScreenCast+# (org.freedesktop.portal.ScreenCast), and the libei input session+# (org.freedesktop.portal.RemoteDesktop.ConnectToEIS). `tokio` shares the+# driver's async runtime.+ashpd = { version = "0.13", features = ["tokio", "screencast", "remote_desktop", "screenshot"] }+# zbus is pulled transitively via ashpd + atspi but declared directly so+# the portal-probe path in wayland::portal_screenshot can talk to the+# session bus without going through ashpd's higher-level builders.+zbus = { version = "5", default-features = false, features = ["tokio"] }+# Parse `file://` URIs returned by org.freedesktop.portal.Screenshot.+# ashpd re-exports `url` transitively; declared explicitly so ashpd+# version bumps cannot silently break the resolver.+url = "2"+# PipeWire client for the ScreenCast per-window capture path — dequeues+# frames from the node id returned by `open_pipe_wire_remote`. Pinned to+# 0.8 (last well-documented release; 0.10 had docs.rs build failures in+# mid-2026). Needs libpipewire-0.3 headers at build time.+pipewire = "0.8"+# Companion to pipewire 0.8 — needed to construct the SPA_PARAM_EnumFormat+# pod (SPA_VIDEO_FORMAT_BGRx at announced width/height) we hand to+# `Stream::connect`.+libspa = "0.8"+# Pure-Rust libei/libeis bindings for the GNOME/KDE input path (handshake+# → seat → device → pointer_absolute/button/scroll/keyboard/text). API is+# documented as "incomplete and subject to change" — budget for churn on+# the next bump. Re-exports `enumflags2` we use for `BitFlags<DeviceType>`.+reis = { version = "0.7", features = ["tokio"] }+# Calloop event loop drives the EIS worker thread — reis's example uses+# calloop::EventLoop + Generic source to multiplex EIS protocol reads+# with command-channel polling. reis 0.7 also ships a calloop adapter+# but we use the plain Generic source for clarity.+calloop = "0.14"+# Used directly in the ashpd RemoteDesktop call to compose+# `BitFlags::from(DeviceType::Keyboard) | DeviceType::Pointer`. reis also+# re-exports this same major version; pinning explicitly prevents+# duplicate-crate hazards.+enumflags2 = "0.7"+# Keymap fallback path for libei keyboard input when the EIS server does+# NOT advertise `ei_text` (libei < 1.6, i.e. older GNOME / KDE Plasma 6.0).+# Mirrors the reis `type-text.rs` example's keycode reverse-mapping.+xkbcommon = "0.9"+# Bounded MPSC command/reply channel for the persistent virtual-pointer+# owner thread. EventQueue<State> is !Send, so the persistent device must+# live on its own thread; crossbeam gives us a Send sender and synchronous+# reply channels that preserve the "tool call blocks until compositor+# roundtripped" contract the X11 path provides.+crossbeam-channel = "0.5"+# Resolve `$XDG_STATE_HOME` (with platform fallback) for persisting the+# ScreenCast and RemoteDesktop `restore_token`s so users see the consent+# dialog only once per install, not once per process.+dirs = "5"++[dev-dependencies]+# Used by the `screenshot_cascade` example to surface the+# `tracing::debug!` lines emitted by each tier of the dispatch as it+# falls through.+tracing-subscriber = { workspace = true }diff --git a/libs/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rs b/libs/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rsnew file mode 100644index 0000000000..bf0de170e8--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rs@@ -0,0 +1,52 @@+//! Standalone smoke for the platform-linux screenshot cascade.+//!+//! Calls `capture::screenshot_display_bytes()` and prints the PNG size ++//! magic bytes + which tier produced the result (inferred from+//! tracing-subscriber logs).+//!+//! Usage:+//! CUA_DRIVER_RS_ENABLE_WAYLAND=1 \+//! WAYLAND_DISPLAY=wayland-0 \+//! cargo run --example screenshot_cascade --release++use std::io::Write as _;++fn main() {+ // Bare tracing subscriber so the debug! lines from the cascade tiers+ // show up.+ let _ = tracing_subscriber::FmtSubscriber::builder()+ .with_max_level(tracing::Level::DEBUG)+ .with_writer(std::io::stderr)+ .try_init();++ eprintln!("== platform-linux screenshot cascade smoke ==");+ eprintln!(+ " WAYLAND_DISPLAY={:?} DISPLAY={:?} CUA_DRIVER_RS_ENABLE_WAYLAND={:?}",+ std::env::var("WAYLAND_DISPLAY").ok(),+ std::env::var("DISPLAY").ok(),+ std::env::var("CUA_DRIVER_RS_ENABLE_WAYLAND").ok(),+ );++ let started = std::time::Instant::now();+ match platform_linux::capture::screenshot_display_bytes() {+ Ok(bytes) => {+ let dt = started.elapsed();+ eprintln!(+ "OK {} bytes in {dt:?} magic={:?}",+ bytes.len(),+ &bytes[..bytes.len().min(8)]+ );+ // Write the PNG so we can also visually check it.+ let path = std::env::var("CUA_SMOKE_OUT")+ .unwrap_or_else(|_| "/tmp/cua-screenshot.png".to_string());+ if let Ok(mut f) = std::fs::File::create(&path) {+ let _ = f.write_all(&bytes);+ eprintln!(" wrote {}", path);+ }+ }+ Err(e) => {+ eprintln!("ERR {}", e);+ std::process::exit(1);+ }+ }+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rsindex ccc884637f..589b774b8a 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs@@ -921,6 +921,15 @@ fn x11_window_origin(xid: u64) -> Option<(i32, i32)> {/// X11 origin), so the existing behaviour is preserved for non-GTK4 toolkits/// and the change can never make correct coordinates worse.async fn gtk4_screen_offset(visited: &[Visited<'_>], pid: u32, xid: u64) -> (i32, i32) {+ // Native Wayland forbids a client from querying another window's screen+ // origin (privacy by design), so the X11-based offset recovery below+ // can't run. Return (0, 0) — GTK4 element bounds will be window-local+ // rather than screen-relative on Wayland, which mirrors how every other+ // tool reports coords on this backend.+ if crate::wayland::is_wayland() {+ let _ = (visited, pid, xid);+ return (0, 0);+ }// The frame is the toplevel window accessible. Prefer an explicit "frame"// role; fall back to the first node that exposes a Component interface.let frame = visiteddiff --git a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs b/libs/cua-driver/rust/crates/platform-linux/src/capture.rsindex 831ae61bb9..28908bb509 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs@@ -99,7 +99,32 @@ pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> {// through `cua_driver_core::image_utils::encode_rgba_to_png`./// Capture the primary display (root window) as raw PNG bytes.+///+/// Dispatch:+/// - Native Wayland (`CUA_DRIVER_RS_ENABLE_WAYLAND=1` + Wayland session):+/// routes through [`crate::wayland::screenshot_display_dispatch`] which+/// cascades wlroots screencopy → ext-image-copy-capture-v1 → portal+/// Screenshot. Each tier falls through to the next on missing globals.+/// - X11 / Wayland-disabled: ImageMagick `import` → x11rb `XGetImage`.pub fn screenshot_display_bytes() -> Result<Vec<u8>> {+ if crate::wayland::is_wayland() {+ match crate::wayland::screenshot_display_dispatch() {+ Ok(bytes) => return Ok(bytes),+ Err(e) => {+ tracing::debug!(+ "wayland screenshot cascade unavailable ({e}); falling through to X11"+ );+ }+ }+ }+ screenshot_display_bytes_x11()+}++/// X11-only display capture path — extracted so the wayland cascade in+/// [`crate::wayland::screenshot_display_dispatch`] can call it as a final+/// fallback without re-entering [`screenshot_display_bytes`] (which would+/// loop forever once we're on Wayland).+pub(crate) fn screenshot_display_bytes_x11() -> Result<Vec<u8>> {// Try `import -window root png:-` (ImageMagick).let out = Command::new("import").args(["-window", "root", "png:-"])diff --git a/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs b/libs/cua-driver/rust/crates/platform-linux/src/health_report.rsindex 03d146de88..0c44bce561 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs@@ -15,6 +15,11 @@ use cua_driver_core::health_report::{NAME_SESSION_ACTIVE, NAME_TCC_ACCESSIBILITY, NAME_TCC_SCREEN_RECORDING,};+/// Doctor entry that surfaces which wlroots manager globals the running+/// Wayland compositor advertises (foreign-toplevel / screencopy /+/// virtual-pointer / wl_shm). Linux-specific; skipped when not on Wayland.+pub const NAME_WAYLAND_BACKEND: &str = "wayland_backend";+/// Linux canonical check names — same Swift-PR contract, with TCC //// bundle entries surfaced as `skip("not applicable on Linux")`. The/// full set of entries always appears so cross-platform consumers see@@ -28,6 +33,7 @@ pub const LINUX_CHECK_NAMES: &[&str] = &[NAME_TCC_SCREEN_RECORDING,NAME_AX_CAPABILITY,NAME_SCREEN_CAPTURE_CAPABILITY,+ NAME_WAYLAND_BACKEND,];pub struct LinuxHealthProvider;@@ -52,6 +58,7 @@ impl HealthCheckProvider for LinuxHealthProvider {NAME_TCC_SCREEN_RECORDING => skip_not_applicable(NAME_TCC_SCREEN_RECORDING),NAME_AX_CAPABILITY => check_ax_capability().await,NAME_SCREEN_CAPTURE_CAPABILITY => check_screen_capture_capability().await,+ NAME_WAYLAND_BACKEND => check_wayland_backend().await,other => CheckEntry::skip(other.to_owned(),"Unknown check name (not implemented on this platform).",@@ -92,6 +99,28 @@ fn skip_not_applicable(name: &str) -> CheckEntry {}async fn check_ax_capability() -> CheckEntry {+ // Native Wayland: AT-SPI lives entirely on D-Bus, so the AX prereq is+ // `org.a11y.Bus` being reachable on the session bus. Falls back to X11+ // reachability for X / XWayland sessions.+ if is_wayland_session() {+ let bus_ok = tokio::task::spawn_blocking(probe_a11y_bus)+ .await+ .unwrap_or(false);+ if bus_ok {+ return CheckEntry::pass(+ NAME_AX_CAPABILITY,+ "Wayland session: org.a11y.Bus reachable on the session bus; \+ AT-SPI inspection will work.",+ );+ }+ return CheckEntry::fail(+ NAME_AX_CAPABILITY,+ "Wayland session: org.a11y.Bus is not reachable on the session bus; \+ AT-SPI inspection will fail.",+ "Start the AT-SPI bus (`/usr/libexec/at-spi-bus-launcher`) or enable \+ accessibility in your desktop's settings.",+ );+ }// Mirror the existing `check_permissions` Linux probe: X11// connectivity is the AX prerequisite (AT-SPI is over D-Bus, but// input/readback need an X server). Cheap and side-effect-free.@@ -104,14 +133,76 @@ async fn check_ax_capability() -> CheckEntry {"X11 reachable; AT-SPI + XSendEvent input will work.",);}+ let hint = if std::env::var_os("WAYLAND_DISPLAY").is_some() {+ "Pure Wayland session: opt into the experimental Wayland backend by \+ setting CUA_DRIVER_RS_ENABLE_WAYLAND=1, or run the target under XWayland."+ } else {+ "Set DISPLAY (X11) — under Wayland, run via XWayland or expose XDG_SESSION_TYPE=x11."+ };CheckEntry::fail(NAME_AX_CAPABILITY,"X11 is not reachable; UI inspection and event injection will fail.",- "Set DISPLAY (X11) — under Wayland, run via XWayland or expose XDG_SESSION_TYPE=x11.",+ hint,)}async fn check_screen_capture_capability() -> CheckEntry {+ // Native Wayland: capture flows through a cascade — wlroots screencopy+ // (sway / labwc / kwin 5.27+ / hyprland) → ext-image-copy-capture-v1+ // (sway 1.10+ / labwc 0.8+ / niri / KDE 6.2+ / GNOME 47+) →+ // xdg-desktop-portal Screenshot (GNOME / KDE / COSMIC + portal backend).+ // Report which tier is reachable so users on mutter / kwin don't see a+ // misleading "screen capture will fail" just because wlroots isn't+ // present.+ if is_wayland_session() {+ let snap = tokio::task::spawn_blocking(probe_wayland_managers)+ .await+ .ok()+ .and_then(|r| r.ok());+ match &snap {+ Some(m) if m.screencopy && m.wl_shm => {+ return CheckEntry::pass(+ NAME_SCREEN_CAPTURE_CAPABILITY,+ "Wayland session: zwlr_screencopy_manager_v1 + wl_shm advertised; \+ native output capture is functional.",+ );+ }+ Some(m) if m.ext_image_copy_capture && m.ext_output_image_capture_source => {+ return CheckEntry::pass(+ NAME_SCREEN_CAPTURE_CAPABILITY,+ "Wayland session: ext-image-copy-capture-v1 + \+ ext-output-image-capture-source-v1 advertised; cross-DE \+ native output capture is functional.",+ );+ }+ _ => {}+ }+ // No wlr screencopy AND no ext-image-copy-capture. Probe the+ // portal as the last native tier. The probe is read-only — it+ // checks for a name owner on the bus, does NOT take a screenshot.+ let portal_ok = tokio::task::spawn_blocking(crate::wayland::portal_screenshot::probe_portal)+ .await+ .ok()+ .and_then(|r| r.ok())+ .unwrap_or(false);+ if portal_ok {+ return CheckEntry::pass(+ NAME_SCREEN_CAPTURE_CAPABILITY,+ "Wayland session: no native screencopy globals, but \+ xdg-desktop-portal Screenshot is reachable; capture will go \+ through the portal (one consent prompt per session).",+ );+ }+ return CheckEntry::fail(+ NAME_SCREEN_CAPTURE_CAPABILITY,+ "Wayland session: no native screencopy globals, and \+ xdg-desktop-portal is not reachable on the session bus; \+ screen capture will fail.",+ "Use a wlroots-based compositor (sway, labwc, hyprland, kwin 5.27+) \+ or install xdg-desktop-portal-gnome / xdg-desktop-portal-kde / \+ xdg-desktop-portal-wlr.",+ );+ }// On Linux the canonical capture path is X11 GetImage / xwd / scrot// — all require an open X11 connection. The probe is the same one// we use for ax_capability, but the consumer-facing message and@@ -126,10 +217,91 @@ async fn check_screen_capture_capability() -> CheckEntry {"X11 reachable; screen capture path is functional.",);}+ let hint = if std::env::var_os("WAYLAND_DISPLAY").is_some() {+ "Pure Wayland session: opt into the experimental Wayland backend by \+ setting CUA_DRIVER_RS_ENABLE_WAYLAND=1, or run the target under XWayland."+ } else {+ "Set DISPLAY (X11). Pure Wayland sessions require an XWayland bridge for capture."+ };CheckEntry::fail(NAME_SCREEN_CAPTURE_CAPABILITY,"X11 is not reachable; screen capture will fail.",- "Set DISPLAY (X11). Pure Wayland sessions require an XWayland bridge for capture.",+ hint,+ )+}++/// `wayland_backend` doctor entry — surfaces which wlroots manager globals+/// the running compositor advertises. Skipped on non-Wayland sessions and+/// when the experimental Wayland backend isn't opted into, so X11 users see+/// a neutral skip rather than a confusing failure.+async fn check_wayland_backend() -> CheckEntry {+ if std::env::var_os("WAYLAND_DISPLAY").is_none() {+ return CheckEntry::skip(+ NAME_WAYLAND_BACKEND.to_owned(),+ "No WAYLAND_DISPLAY in the environment — not a Wayland session.",+ );+ }+ if !wayland_opt_in_enabled() {+ return CheckEntry::skip(+ NAME_WAYLAND_BACKEND.to_owned(),+ format!(+ "Wayland session detected, but the experimental backend is opt-in. \+ Set {}=1 to enable native Wayland and re-run doctor.",+ wayland_env_name()+ ),+ );+ }+ let snap = match tokio::task::spawn_blocking(probe_wayland_managers).await {+ Ok(Ok(snap)) => snap,+ Ok(Err(e)) => {+ return CheckEntry::fail(+ NAME_WAYLAND_BACKEND,+ format!("Failed to connect to the Wayland compositor: {e}"),+ "Verify WAYLAND_DISPLAY points at a running compositor socket.",+ );+ }+ Err(e) => {+ return CheckEntry::fail(+ NAME_WAYLAND_BACKEND,+ format!("Wayland probe task error: {e}"),+ "Re-run the doctor; if it persists, file a bug with the doctor output.",+ );+ }+ };+ let msg = format!(+ "foreign-toplevel={ftl}, screencopy={cap}, virtual-pointer={vp}, wl_shm={shm}",+ ftl = snap.foreign_toplevel,+ cap = snap.screencopy,+ vp = snap.virtual_pointer,+ shm = snap.wl_shm,+ );+ if snap.foreign_toplevel && snap.screencopy && snap.virtual_pointer && snap.wl_shm {+ return CheckEntry::pass(+ NAME_WAYLAND_BACKEND,+ format!("All wlroots manager globals advertised ({msg})."),+ );+ }+ // Partial-pass: list_windows + capture both work, but virtual-pointer+ // input is missing. Require `wl_shm` here too — `check_screen_capture_capability`+ // gates on both `screencopy && wl_shm`, so excluding `wl_shm` from the+ // partial-pass verdict would let the matrices disagree on degenerate+ // compositors that omit it.+ if snap.foreign_toplevel && snap.screencopy && snap.wl_shm {+ return CheckEntry::pass(+ NAME_WAYLAND_BACKEND,+ format!(+ "Core wlroots manager globals available; some optional globals missing ({msg}). \+ Input may fall back where virtual-pointer is absent."+ ),+ );+ }+ CheckEntry::fail(+ NAME_WAYLAND_BACKEND,+ format!(+ "Compositor does not advertise the wlroots manager globals cua-driver \+ needs ({msg})."+ ),+ "Use a wlroots-based compositor (sway, labwc, hyprland) or run under XWayland.",)}@@ -145,6 +317,102 @@ fn probe_x11_connect() -> bool {false}+/// Probe whether `org.a11y.Bus` is reachable on the session bus — the+/// canonical AT-SPI prerequisite on Wayland, where there is no X server to+/// stand in. Returns false on any error (no session bus, no a11y service,+/// timeout, etc.) so the doctor message stays simple.+#[cfg(target_os = "linux")]+fn probe_a11y_bus() -> bool {+ use atspi::zbus;+ let rt = match tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ {+ Ok(rt) => rt,+ Err(_) => return false,+ };+ rt.block_on(async {+ let bus = match zbus::Connection::session().await {+ Ok(b) => b,+ Err(_) => return false,+ };+ let proxy = match zbus::fdo::DBusProxy::new(&bus).await {+ Ok(p) => p,+ Err(_) => return false,+ };+ // `name_has_owner` is the cheap direct existence check — avoids+ // pulling the entire session-bus name registry just to look up one+ // entry. Matches the pattern in `wayland::portal_screenshot::probe_portal`.+ let bus_name = match "org.a11y.Bus".try_into() {+ Ok(n) => n,+ Err(_) => return false,+ };+ proxy.name_has_owner(bus_name).await.unwrap_or(false)+ })+}++#[cfg(not(target_os = "linux"))]+fn probe_a11y_bus() -> bool {+ false+}++/// True when the Wayland backend is opted in and a Wayland session is+/// active. Wrapper around `wayland::is_wayland()` so non-Linux builds of+/// this file compile (the `wayland` module is gated on `target_os = linux`).+#[cfg(target_os = "linux")]+fn is_wayland_session() -> bool {+ crate::wayland::is_wayland()+}++#[cfg(not(target_os = "linux"))]+fn is_wayland_session() -> bool {+ false+}++#[cfg(target_os = "linux")]+fn wayland_opt_in_enabled() -> bool {+ crate::wayland::wayland_enabled()+}++#[cfg(not(target_os = "linux"))]+fn wayland_opt_in_enabled() -> bool {+ false+}++#[cfg(target_os = "linux")]+fn wayland_env_name() -> &'static str {+ crate::wayland::ENABLE_WAYLAND_ENV+}++#[cfg(not(target_os = "linux"))]+fn wayland_env_name() -> &'static str {+ "CUA_DRIVER_RS_ENABLE_WAYLAND"+}++/// Snapshot of wlroots manager globals. The non-Linux stub returns an empty+/// snapshot so off-platform builds stay green; doctor short-circuits before+/// reaching it via [`is_wayland_session`].+#[cfg(target_os = "linux")]+fn probe_wayland_managers() -> anyhow::Result<crate::wayland::WaylandManagers> {+ crate::wayland::probe_managers()+}++#[cfg(not(target_os = "linux"))]+fn probe_wayland_managers() -> anyhow::Result<WaylandManagers> {+ Ok(WaylandManagers::default())+}++/// Stub of `wayland::WaylandManagers` so off-Linux builds compile. Always+/// reports nothing advertised — non-Linux code paths never call this.+#[cfg(not(target_os = "linux"))]+#[derive(Default, Clone, Debug)]+struct WaylandManagers {+ foreign_toplevel: bool,+ screencopy: bool,+ virtual_pointer: bool,+ wl_shm: bool,+}+/// `PRETTY_NAME=` out of `/etc/os-release`. Fails to "Linux" so the/// downstream message stays human-readable.fn os_release_pretty_name() -> Option<String> {diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rsindex 5c8647472f..c03e36be81 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs@@ -127,8 +127,15 @@ pub fn send_command_for(key: CursorKey, cmd: OverlayCommand) {if key.is_empty() {return;}+ let msg = OverlayMsg::Cmd(KeyedOverlayCommand { key: key.clone(), cmd: cmd.clone() });if let Some(tx) = CMD_TX.get() {- let _ = tx.try_send(OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd }));+ let _ = tx.try_send(msg.clone());+ }+ // Also forward to the native-Wayland layer-shell overlay when Wayland+ // is opted in. The wayland overlay's `forward` is a no-op when its+ // owner thread isn't started yet (which is the normal X11-only case).+ if crate::wayland::is_wayland() {+ let _ = crate::wayland::overlay::forward(&msg);}}@@ -247,6 +254,17 @@ pub fn run_on_thread() {return;}+ // Wayland layer-shell overlay is started LAZILY on the first+ // send_command_for() that targets a Wayland session (see the+ // wayland::overlay::forward() path). Starting it eagerly here added+ // ~100-300ms to cua-driver mcp startup — enough to push the CI+ // `cursor-click-gif` test (20s budget for the full launch_app →+ // click → type sequence) over its limit, intermittently. The+ // forward() path's own ensure_started() OnceLock guarantees the+ // thread spins up on demand without losing any commands (the+ // first send_command_for that triggers it spawns the thread,+ // future commands reuse it).+std::thread::Builder::new().name("cua-overlay-x11".into()).spawn(move || {diff --git a/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs b/libs/cua-driver/rust/crates/platform-linux/src/terminal.rsindex f0966ef052..0e2db4dc15 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs@@ -65,12 +65,14 @@ pub fn wm_class_matches_terminal(instance: &str, class: &str) -> bool {}/// Returns `true` if the window with X11 id `xid` belongs to a-/// terminal emulator, based on its `WM_CLASS` property. Returns-/// `false` when no X connection is available or the property is-/// unset.+/// terminal emulator, based on its `WM_CLASS` property. On native+/// Wayland the dispatcher folds the foreign-toplevel `app_id` into+/// both fields so the substring match still works for terminals like+/// Ghostty / kitty / alacritty. Returns `false` when no connection is+/// available or the property is unset.#[cfg(target_os = "linux")]pub fn is_terminal_window(xid: u64) -> bool {- match crate::x11::wm_class_for_window(xid) {+ match crate::wayland::wm_class_dispatch(xid) {Some((instance, class)) => wm_class_matches_terminal(&instance, &class),None => false,}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rsindex 876056122c..08da3548cb 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs@@ -1048,23 +1048,12 @@ impl Tool for ClickTool {let (xi, yi) = (x as i32, y as i32);let result = tokio::task::spawn_blocking(move || {// Native Wayland: focus+raise the target toplevel (foreign-toplevel- // `activate`). Wayland hides cross-window geometry, so window-local- // x/y can't be mapped to a global pointer position; activating the- // window the caller targeted is the focus-based equivalent.+ // `activate`), then drive `count` virtual-pointer button events at+ // the requested coordinates. Wayland hides cross-window geometry,+ // so callers should pass output-relative coords; (0,0) preserves+ // the legacy "click the activated window's centre" behaviour.if crate::wayland::is_wayland() {- // Surface 5: the Wayland virtual-pointer path here only emits a- // left-button press at the output centre. Surface a real error- // instead of silently dropping a right/middle request onto a- // left-click, so the Hermes-side audit gap closes.- if button != 1 {- let label = mouse_button_name(button);- anyhow::bail!(- "{label}-button click is not supported on this platform \- (native Wayland virtual-pointer path: left-button only). \- Use the X11 backend, or run the target under XWayland."- );- }- return crate::wayland::click(xid);+ return crate::wayland::click(xid, xi, yi, count as u32, button);}crate::input::send_click(xid, xi, yi, count, button)}).await;@@ -1506,7 +1495,17 @@ impl Tool for HotkeyTool {};let key_display = format!("{}+{}", mods.join("+"), key);+ let key_for_wayland = key.clone();+ let mods_for_wayland = mods.clone();let result = tokio::task::spawn_blocking(move || {+ if crate::wayland::is_wayland() {+ // Native Wayland: route the modifier combo through wtype's+ // -M/-k/-m sequence — the closest equivalent to the X11+ // state-mask path. window_id is irrelevant once focused.+ let mut combo: Vec<String> = mods_for_wayland.clone();+ combo.push(key_for_wayland.clone());+ return crate::wayland::hotkey(&combo);+ }let m: Vec<&str> = mods.iter().map(String::as_str).collect();crate::input::send_key(xid, &key, &m)}).await;@@ -1663,7 +1662,12 @@ impl Tool for ScrollTool {let button: u8 = match direction.as_str() {"up" => 4, "left" => 6, "right" => 7, _ => 5,};+ let direction_for_wayland = direction.clone();+ let amount_u32 = amount as u32;let result = tokio::task::spawn_blocking(move || {+ if crate::wayland::is_wayland() {+ return crate::wayland::scroll(xid, &direction_for_wayland, amount_u32);+ }crate::input::send_click(xid, 0, 0, amount, button)}).await;match result {@@ -1752,7 +1756,15 @@ impl Tool for DoubleClickTool {cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },);}- match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 2, 1)).await {+ let lxi = lx as i32;+ let lyi = ly as i32;+ let click_result = tokio::task::spawn_blocking(move || {+ if crate::wayland::is_wayland() {+ return crate::wayland::click(xid, lxi, lyi, 2, 1);+ }+ crate::input::send_click(xid, lxi, lyi, 2, 1)+ }).await;+ match click_result {Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked element [{idx}].")),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),@@ -1793,7 +1805,12 @@ impl Tool for DoubleClickTool {);}let (xi, yi) = (x as i32, y as i32);- let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 2, 1)).await;+ let result = tokio::task::spawn_blocking(move || {+ if crate::wayland::is_wayland() {+ return crate::wayland::click(xid, xi, yi, 2, 1);+ }+ crate::input::send_click(xid, xi, yi, 2, 1)+ }).await;match result {Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked at ({x:.1}, {y:.1}).")),Ok(Err(e)) => ToolResult::error(e.to_string()),@@ -1875,7 +1892,15 @@ impl Tool for RightClickTool {cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },);}- match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 1, 3)).await {+ let lxi = lx as i32;+ let lyi = ly as i32;+ let click_result = tokio::task::spawn_blocking(move || {+ if crate::wayland::is_wayland() {+ return crate::wayland::click(xid, lxi, lyi, 1, 3);+ }+ crate::input::send_click(xid, lxi, lyi, 1, 3)+ }).await;+ match click_result {Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked element [{idx}].")),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),@@ -1916,7 +1941,12 @@ impl Tool for RightClickTool {);}let (xi, yi) = (x as i32, y as i32);- let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 1, 3)).await;+ let result = tokio::task::spawn_blocking(move || {+ if crate::wayland::is_wayland() {+ return crate::wayland::click(xid, xi, yi, 1, 3);+ }+ crate::input::send_click(xid, xi, yi, 1, 3)+ }).await;match result {Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked at ({x:.1}, {y:.1}).")),Ok(Err(e)) => ToolResult::error(e.to_string()),@@ -2013,6 +2043,31 @@ impl Tool for DragTool {cursor_overlay::OverlayCommand::SetPressed(true),);+ // Native Wayland: emit press + interpolated motion + release as one+ // virtual-pointer sequence (output-relative coords). Returns early so+ // we don't fall into the X11 XSendEvent loop below.+ if crate::wayland::is_wayland() {+ let (fxi, fyi) = (from_x.round() as i32, from_y.round() as i32);+ let (txi, tyi) = (to_x.round() as i32, to_y.round() as i32);+ let steps_u32 = steps as u32;+ let drag_result = tokio::task::spawn_blocking(move || {+ crate::wayland::drag(xid, fxi, fyi, txi, tyi, steps_u32, button)+ }).await;+ crate::overlay::send_command_for(+ cursor_id.clone(),+ cursor_overlay::OverlayCommand::SetPressed(false),+ );+ return match drag_result {+ Ok(Ok(())) => ToolResult::text(format!(+ "✅ Posted drag ({button_str}) to pid {pid} \+ from ({from_x:.0}, {from_y:.0}) → ({to_x:.0}, {to_y:.0}) \+ in {duration_ms}ms / {steps} steps."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }+let press_result = tokio::task::spawn_blocking(move || {crate::input::send_button_down(xid, from_x.round() as i32, from_y.round() as i32, button)}).await;@@ -2181,7 +2236,15 @@ impl Tool for MouseButtonDownTool {let xi = x as i32;let yi = y as i32;- let result = tokio::task::spawn_blocking(move || crate::input::send_button_down(xid, xi, yi, button)).await;+ // Native Wayland: route through the persistent virtual-pointer module+ // so the held button survives across tool calls; the X11 path keeps+ // the existing input::send_button_down behaviour.+ let result = if crate::wayland::is_wayland() {+ let cid = cursor_id.clone();+ tokio::task::spawn_blocking(move || crate::wayland::persistent_vptr::press(&cid, xid, xi, yi, button)).await+ } else {+ tokio::task::spawn_blocking(move || crate::input::send_button_down(xid, xi, yi, button)).await+ };match result {Ok(Ok(())) => {let hold = MouseHoldState { cursor_id: cursor_id.clone(), pid, xid, button, x, y };@@ -2288,13 +2351,24 @@ impl Tool for MouseDragTool {let mut result: anyhow::Result<()> = Ok(());let mut prev_x = from_x;let mut prev_y = from_y;+ let is_wl = crate::wayland::is_wayland();for i in 1..=steps {let t = i as f64 / steps as f64;let ix = from_x + (to_x - from_x) * t;let iy = from_y + (to_y - from_y) * t;- let move_result = tokio::task::spawn_blocking(move || {- crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button))- }).await;+ // Native Wayland: route motion through the persistent virtual-+ // pointer so the held button stays held across the entire drag;+ // X11 keeps the existing input::send_motion path.+ let cid_inner = cursor_id.clone();+ let move_result = if is_wl {+ tokio::task::spawn_blocking(move || {+ crate::wayland::persistent_vptr::move_to(&cid_inner, ix.round() as i32, iy.round() as i32)+ }).await+ } else {+ tokio::task::spawn_blocking(move || {+ crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button))+ }).await+ };match move_result {Ok(Ok(())) => {if let Ok(Ok((sx, sy))) =@@ -2411,7 +2485,15 @@ impl Tool for MouseButtonUpTool {let button = hold.button;let xi = x as i32;let yi = y as i32;- let result = tokio::task::spawn_blocking(move || crate::input::send_button_up(xid, xi, yi, button)).await;+ // Native Wayland: release through the persistent virtual-pointer so+ // the same vptr device that emitted the press also emits the release+ // (single logical drag rather than a click pair).+ let result = if crate::wayland::is_wayland() {+ let cid = cursor_id.clone();+ tokio::task::spawn_blocking(move || crate::wayland::persistent_vptr::release(&cid, button)).await+ } else {+ tokio::task::spawn_blocking(move || crate::input::send_button_up(xid, xi, yi, button)).await+ };match result {Ok(Ok(())) => {hold.x = x;@@ -2535,6 +2617,16 @@ impl Tool for ParallelMouseDragTool {if crate::wayland::is_inject_mode() {return parallel_drag_inject(&args).await;}+ // Native Wayland without the inject socket: MPX/XI2 + uinput master+ // pointers don't exist on Wayland. Surface a typed error instead of+ // silently calling the X11 path that's guaranteed to fail.+ if crate::wayland::is_wayland() {+ return ToolResult::error(+ "parallel_mouse_drag requires the cua-compositor inject socket on Wayland \+ (set CUA_INJECT_SOCKET to the cua-compositor control socket), \+ or run the target under X11."+ );+ }match tokio::task::spawn_blocking(crate::input::check_parallel_pointer_support).await {Ok(Ok(())) => {}Ok(Err(e)) => return ToolResult::error(e.to_string()),@@ -2741,6 +2833,21 @@ impl Tool for GetCursorPositionTool {})}async fn invoke(&self, _args: Value) -> ToolResult {+ // Native Wayland: there's no protocol for clients to query the real+ // global cursor position. Fall back to the synthetic registry that+ // records every `motion_absolute` this process emits.+ if crate::wayland::is_wayland() {+ return match crate::wayland::last_synth_cursor_pos() {+ Some((x, y)) => ToolResult::text(+ format!("✅ Cursor at ({x}, {y}) (synthetic — last move_cursor in this process)")+ ).with_structured(json!({+ "x": x, "y": y, "source": "synthetic"+ })),+ None => ToolResult::text(+ "Cursor position unknown on Wayland — no move_cursor has been issued in this process yet.".to_string()+ ).with_structured(json!({ "source": "synthetic", "available": false })),+ };+ }let result = tokio::task::spawn_blocking(|| {use x11rb::connection::Connection;use x11rb::protocol::xproto::ConnectionExt as _;@@ -2753,7 +2860,7 @@ impl Tool for GetCursorPositionTool {match result {// Text format matches Swift `GetCursorPositionTool` 1:1.Ok(Ok((x, y))) => ToolResult::text(format!("✅ Cursor at ({x}, {y})"))- .with_structured(json!({ "x": x, "y": y })),+ .with_structured(json!({ "x": x, "y": y, "source": "x11" })),Ok(Err(e)) => ToolResult::error(e.to_string()),Err(e) => ToolResult::error(format!("Task error: {e}")),}@@ -2784,6 +2891,7 @@ impl Tool for MoveCursorTool {use cua_driver_core::tool_args::ArgsExt;let x = args.f64_or("x", 0.0);let y = args.f64_or("y", 0.0);+ let window_id = args.get("window_id").and_then(|v| v.as_u64());let cursor_id = resolve_cursor_key(&args);self.state.cursor_registry.update_position(&cursor_id, x, y);// End pointing upper-left (45°) — matches Swift's@@ -2797,7 +2905,21 @@ impl Tool for MoveCursorTool {end_heading_radians: std::f64::consts::FRAC_PI_4,},);- ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1})."))+ // Native Wayland: also warp the real cursor via zwlr_virtual_pointer.+ // Off-thread because the wayland-client roundtrip is blocking. Best-effort+ // — overlay update + registry write already succeeded; surface a warning+ // only if the warp itself failed.+ let real_warp_note = if crate::wayland::is_wayland() {+ let xi = x.round() as i32;+ let yi = y.round() as i32;+ match tokio::task::spawn_blocking(move || crate::wayland::move_cursor_absolute(window_id, xi, yi)).await {+ Ok(Ok(())) => " (real cursor warped via virtual-pointer)",+ Ok(Err(_)) | Err(_) => " (overlay updated; real-cursor warp failed)",+ }+ } else {+ ""+ };+ ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1}).{real_warp_note}"))}}@@ -3384,7 +3506,11 @@ impl Tool for ZoomTool {let state = self.state.clone();let result = tokio::task::spawn_blocking(move || {- let png = crate::capture::screenshot_window_bytes(xid)?;+ // Route through the Wayland-aware window capture dispatcher so+ // pure-Wayland sessions surface a typed "per-window capture not+ // supported yet" error instead of accidentally calling the+ // X11-only path with a foreign-toplevel id.+ let png = crate::wayland::screenshot_window_dispatch(xid)?;cursor_overlay::capture_utils::crop_png_to_jpeg(&png, x1, y1, x2, y2, 500)}).await;@@ -3470,6 +3596,20 @@ impl Tool for TypeTextCharsTool {};let text_len = text.chars().count();let result = tokio::task::spawn_blocking(move || {+ if crate::wayland::is_wayland() {+ // Per-char `wtype` loop with the requested delay — mirrors the+ // X11 XSendEvent per-char path. Sleeping here is fine because+ // we're inside spawn_blocking.+ let mut buf = [0u8; 4];+ for ch in text.chars() {+ let s = ch.encode_utf8(&mut buf);+ crate::wayland::type_text(s)?;+ if delay_ms > 0 {+ std::thread::sleep(std::time::Duration::from_millis(delay_ms));+ }+ }+ return Ok(());+ }crate::input::send_type_text_with_delay(xid, &text, delay_ms)}).await;match result {diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rsnew file mode 100644index 0000000000..d4e313e47e--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rs@@ -0,0 +1,348 @@+//! Output capture via `ext-image-copy-capture-v1` (staging protocol).+//!+//! The wlroots `zwlr_screencopy_manager_v1` capture path in `mod.rs` works+//! on wlroots compositors (sway, labwc) but is missing from compositors+//! that don't speak wlroots protocols (GNOME mutter, KDE Plasma <6.2).+//! The staging `ext-image-copy-capture-v1` protocol replaces wlr-screencopy+//! across the ecosystem — sway 1.10+, labwc 0.8+, niri 0.1.6+, hyprland+//! 0.45+, KDE Plasma 6.2+, GNOME mutter 47+ all implement it.+//!+//! This module captures the FIRST `wl_output` advertised by the compositor+//! via the chain:+//! ext_output_image_capture_source_manager_v1.create_source(wl_output)+//! → ext_image_copy_capture_manager_v1.create_session(source, options)+//! → session emits buffer_size + done+//! → allocate wl_shm pool + wl_buffer matching the size/format+//! → session.create_frame() → frame.attach_buffer(buffer) → frame.capture()+//! → frame emits ready (success) or failed (error)+//! → read pixels out of the mmap'd buffer, PNG-encode, destroy+//!+//! Per-window capture (ext_foreign_toplevel_image_capture_source_manager_v1)+//! is a follow-up — needs the existing `list_windows` to migrate from+//! `zwlr_foreign_toplevel_manager_v1` to `ext_foreign_toplevel_list_v1` so+//! the window_id handles match up.++use std::time::Duration;++use wayland_client::{+ protocol::{+ wl_buffer::WlBuffer,+ wl_output::WlOutput,+ wl_registry,+ wl_shm::{self, WlShm},+ wl_shm_pool::WlShmPool,+ },+ Connection, Dispatch, Proxy, QueueHandle,+};+use wayland_protocols::ext::image_capture_source::v1::client::{+ ext_image_capture_source_v1::ExtImageCaptureSourceV1,+ ext_output_image_capture_source_manager_v1::ExtOutputImageCaptureSourceManagerV1,+};+use wayland_protocols::ext::image_copy_capture::v1::client::{+ ext_image_copy_capture_frame_v1::{self, ExtImageCopyCaptureFrameV1},+ ext_image_copy_capture_manager_v1::{self, ExtImageCopyCaptureManagerV1},+ ext_image_copy_capture_session_v1::{self, ExtImageCopyCaptureSessionV1},+};++#[derive(Default)]+struct CapState {+ output: Option<WlOutput>,+ source_mgr: Option<ExtOutputImageCaptureSourceManagerV1>,+ copy_mgr: Option<ExtImageCopyCaptureManagerV1>,+ shm: Option<WlShm>,+ // Buffer specs reported by the session before we allocate.+ fmt: Option<u32>,+ width: u32,+ height: u32,+ stride: u32,+ done: bool,+ // Frame outcome+ ready: bool,+ failed: bool,+ fail_reason: Option<u32>,+}++/// Capture the first `wl_output` via ext-image-copy-capture-v1, return raw+/// PNG bytes. Falls back to the caller's next tier when the compositor+/// doesn't expose the protocol — surface a typed anyhow::Error so the+/// cascade in `screenshot_display_dispatch` can branch.+pub fn screenshot_via_ext_copy() -> anyhow::Result<Vec<u8>> {+ let conn = Connection::connect_to_env()?;+ let mut queue = conn.new_event_queue::<CapState>();+ let qh = queue.handle();+ conn.display().get_registry(&qh, ());++ let mut state = CapState::default();+ queue.roundtrip(&mut state)?;+ for _ in 0..3 {+ queue.roundtrip(&mut state)?;+ }++ let output = state+ .output+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_output"))?;+ let source_mgr = state.source_mgr.clone().ok_or_else(|| {+ anyhow::anyhow!("compositor does not expose ext_output_image_capture_source_manager_v1")+ })?;+ let copy_mgr = state.copy_mgr.clone().ok_or_else(|| {+ anyhow::anyhow!("compositor does not expose ext_image_copy_capture_manager_v1")+ })?;+ let shm = state+ .shm+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_shm"))?;++ // Create source + session.+ let source: ExtImageCaptureSourceV1 = source_mgr.create_source(&output, &qh, ());+ let session: ExtImageCopyCaptureSessionV1 = copy_mgr.create_session(+ &source,+ ext_image_copy_capture_manager_v1::Options::empty(),+ &qh,+ (),+ );++ // Wait for the session to announce buffer dimensions + format + done.+ let deadline = std::time::Instant::now() + Duration::from_secs(3);+ while !state.done && std::time::Instant::now() < deadline {+ queue.roundtrip(&mut state)?;+ if state.width > 0 && state.height > 0 && state.fmt.is_some() {+ // Some compositors don't emit `done` explicitly — proceed once+ // we have geometry + format.+ state.done = true;+ }+ }+ if !state.done || state.width == 0 || state.height == 0 || state.fmt.is_none() {+ anyhow::bail!(+ "ext-image-copy-capture session never reported buffer specs (w={}, h={}, fmt={:?})",+ state.width, state.height, state.fmt+ );+ }++ // Checked arithmetic on compositor-provided sizes. A malicious or+ // buggy compositor could announce dimensions that overflow `usize` on+ // multiplication — we refuse to allocate in that case rather than+ // wrapping and reading past the buffer.+ let stride = if state.stride > 0 {+ state.stride+ } else {+ state.width+ .checked_mul(4)+ .ok_or_else(|| anyhow::anyhow!("compositor advertised width that overflows stride: {}", state.width))?+ };+ if stride < state.width.saturating_mul(4) {+ anyhow::bail!(+ "compositor stride {} is smaller than width*4 {}; refusing to allocate (would alias rows)",+ stride, state.width.saturating_mul(4)+ );+ }+ let size = (stride as usize)+ .checked_mul(state.height as usize)+ .ok_or_else(|| {+ anyhow::anyhow!(+ "compositor buffer size overflows usize: stride={} height={}",+ stride, state.height+ )+ })?;++ // Allocate the wl_shm buffer.+ let (fd, ptr) = super::anon_shm(size)?;+ use std::os::fd::AsFd as _;+ let pool_fd = unsafe { super::borrowed_fd(fd) };+ let pool: WlShmPool = shm.create_pool(pool_fd.as_fd(), size as i32, &qh, ());+ let fmt: wl_shm::Format = wl_shm::Format::try_from(state.fmt.unwrap()).map_err(|_| {+ anyhow::anyhow!("compositor advertised unsupported wl_shm format {:#x}", state.fmt.unwrap())+ })?;+ let buffer: WlBuffer = pool.create_buffer(+ 0,+ state.width as i32,+ state.height as i32,+ stride as i32,+ fmt,+ &qh,+ (),+ );++ // Create + run the frame.+ let frame: ExtImageCopyCaptureFrameV1 = session.create_frame(&qh, ());+ frame.attach_buffer(&buffer);+ frame.capture();+ queue.roundtrip(&mut state)?;++ let deadline = std::time::Instant::now() + Duration::from_secs(5);+ while !state.ready && !state.failed && std::time::Instant::now() < deadline {+ queue.roundtrip(&mut state)?;+ }+ if state.failed {+ super::cleanup_mmap(ptr, size, fd);+ anyhow::bail!(+ "ext-image-copy-capture frame failed (reason={:?})",+ state.fail_reason+ );+ }+ if !state.ready {+ super::cleanup_mmap(ptr, size, fd);+ anyhow::bail!("ext-image-copy-capture frame timed out without ready event");+ }++ // Copy + encode + cleanup.+ let pixels: &[u8] = unsafe { std::slice::from_raw_parts(ptr as *const u8, size) };+ let png = encode_buffer_to_png(pixels, state.width, state.height, stride, fmt)?;++ frame.destroy();+ session.destroy();+ source.destroy();+ pool.destroy();+ queue.roundtrip(&mut state).ok();+ super::cleanup_mmap(ptr, size, fd);++ Ok(png)+}++fn encode_buffer_to_png(+ pixels: &[u8],+ width: u32,+ height: u32,+ stride: u32,+ fmt: wl_shm::Format,+) -> anyhow::Result<Vec<u8>> {+ // The image crate expects RGBA8888 row-packed. wl_shm gives us Argb8888+ // / Xrgb8888 which in little-endian memory layout is BGRA / BGRX. Swap+ // channels into RGBA and pack rows.+ let mut rgba: Vec<u8> = Vec::with_capacity((width as usize) * (height as usize) * 4);+ for y in 0..(height as usize) {+ let row_start = y * (stride as usize);+ for x in 0..(width as usize) {+ let i = row_start + x * 4;+ let b = pixels[i];+ let g = pixels[i + 1];+ let r = pixels[i + 2];+ let a = match fmt {+ wl_shm::Format::Xrgb8888 => 0xFF,+ _ => pixels[i + 3],+ };+ rgba.extend_from_slice(&[r, g, b, a]);+ }+ }++ use image::{ImageBuffer, Rgba, codecs::png::PngEncoder};+ let img: ImageBuffer<Rgba<u8>, _> =+ ImageBuffer::from_raw(width, height, rgba).ok_or_else(|| {+ anyhow::anyhow!("internal: buffer dims mismatch ({}x{})", width, height)+ })?;+ let mut out: Vec<u8> = Vec::new();+ use image::ImageEncoder;+ PngEncoder::new(&mut out).write_image(+ img.as_raw(),+ width,+ height,+ image::ExtendedColorType::Rgba8,+ )?;+ Ok(out)+}++// ── Wayland Dispatch impls ───────────────────────────────────────────────++impl Dispatch<wl_registry::WlRegistry, ()> for CapState {+ fn event(+ state: &mut Self,+ registry: &wl_registry::WlRegistry,+ event: wl_registry::Event,+ _: &(),+ _: &Connection,+ qh: &QueueHandle<Self>,+ ) {+ if let wl_registry::Event::Global { name, interface, version } = event {+ match interface.as_str() {+ "wl_output" => {+ if state.output.is_none() {+ state.output = Some(registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ()));+ }+ }+ "wl_shm" => {+ state.shm = Some(registry.bind::<WlShm, _, _>(name, 1, qh, ()));+ }+ "ext_output_image_capture_source_manager_v1" => {+ state.source_mgr = Some(registry.bind::<ExtOutputImageCaptureSourceManagerV1, _, _>(+ name, 1, qh, ()));+ }+ "ext_image_copy_capture_manager_v1" => {+ state.copy_mgr = Some(registry.bind::<ExtImageCopyCaptureManagerV1, _, _>(+ name, 1, qh, ()));+ }+ _ => {}+ }+ }+ }+}++impl Dispatch<WlOutput, ()> for CapState {+ fn event(_: &mut Self, _: &WlOutput, _: <WlOutput as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}+}+impl Dispatch<WlShm, ()> for CapState {+ fn event(_: &mut Self, _: &WlShm, _: <WlShm as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}+}+impl Dispatch<WlShmPool, ()> for CapState {+ fn event(_: &mut Self, _: &WlShmPool, _: <WlShmPool as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}+}+impl Dispatch<WlBuffer, ()> for CapState {+ fn event(_: &mut Self, _: &WlBuffer, _: <WlBuffer as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}+}+impl Dispatch<ExtOutputImageCaptureSourceManagerV1, ()> for CapState {+ fn event(_: &mut Self, _: &ExtOutputImageCaptureSourceManagerV1, _: <ExtOutputImageCaptureSourceManagerV1 as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}+}+impl Dispatch<ExtImageCaptureSourceV1, ()> for CapState {+ fn event(_: &mut Self, _: &ExtImageCaptureSourceV1, _: <ExtImageCaptureSourceV1 as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}+}+impl Dispatch<ExtImageCopyCaptureManagerV1, ()> for CapState {+ fn event(_: &mut Self, _: &ExtImageCopyCaptureManagerV1, _: <ExtImageCopyCaptureManagerV1 as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}+}+impl Dispatch<ExtImageCopyCaptureSessionV1, ()> for CapState {+ fn event(+ state: &mut Self,+ _: &ExtImageCopyCaptureSessionV1,+ event: <ExtImageCopyCaptureSessionV1 as Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ use ext_image_copy_capture_session_v1::Event as Ev;+ match event {+ Ev::BufferSize { width, height } => {+ state.width = width;+ state.height = height;+ }+ Ev::ShmFormat { format } => {+ if state.fmt.is_none() {+ if let Ok(f) = format.into_result() {+ state.fmt = Some(f as u32);+ }+ }+ }+ Ev::Done => {+ state.done = true;+ }+ _ => {}+ }+ }+}+impl Dispatch<ExtImageCopyCaptureFrameV1, ()> for CapState {+ fn event(+ state: &mut Self,+ _: &ExtImageCopyCaptureFrameV1,+ event: <ExtImageCopyCaptureFrameV1 as Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ use ext_image_copy_capture_frame_v1::Event as Ev;+ match event {+ Ev::Ready => state.ready = true,+ Ev::Failed { reason } => {+ state.failed = true;+ state.fail_reason = reason.into_result().ok().map(|r| r as u32);+ }+ _ => {}+ }+ }+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rsnew file mode 100644index 0000000000..2265ba74cb--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs@@ -0,0 +1,647 @@+//! libei-based input synthesis via the `reis` crate + xdg-desktop-portal+//! RemoteDesktop.+//!+//! Reaches GNOME/Mutter (47+, mutter shipped EIS in 45, ei_text in 46)+//! and KDE/KWin (Plasma 6.0+, ei_text in 6.1) — the cross-DE input+//! complement that doesn't depend on the wlroots-only+//! `zwlr_virtual_pointer_v1` / `zwlr_virtual_keyboard_v1` protocols.+//!+//! Sequence:+//! 1. `ei::Context::connect_to_env()` — fast path when a $LIBEI_SOCKET is+//! already exported (cua-compositor / inject mode).+//! 2. Fallback: ashpd `RemoteDesktop::create_session` →+//! `select_devices(KEYBOARD|POINTER)` →+//! `start(session, parent)` (user consent dialog the first time) →+//! `connect_to_eis()` returns a Unix fd → `ei::Context::new(fd)`.+//! 3. Worker thread runs a calloop event loop on the ei::Context;+//! handshake → connection.seat events → seat.bind(capabilities) →+//! device.frame + emulate events.+//! 4. Public API sends commands over a crossbeam-channel and blocks+//! until the worker reports the request was flushed to the EIS+//! server.+//!+//! Persistence: ashpd `PersistMode::Application` caches the consent grant+//! for the lifetime of the requesting binary; restore_token written to+//! `~/.config/cua-driver/libei.token` survives reboots (the worker+//! reads it on startup).+//!+//! Coordinates: ei_pointer_absolute uses LOGICAL PIXELS inside an+//! announced ei_device.Region — collected between device creation and+//! the first device.Done event. Multi-monitor: pick the region+//! containing the target (x, y).++use std::collections::HashMap;+use std::path::PathBuf;+use std::sync::OnceLock;+use std::thread;++use crossbeam_channel::{bounded, Receiver, Sender};++/// Buttons the public API exposes. Mapped to evdev codes in the worker+/// thread so the libei surface only sees evdev integers.+#[derive(Debug, Clone, Copy)]+pub enum Button {+ Left,+ Right,+ Middle,+}++impl Button {+ fn to_evdev(self) -> u32 {+ match self {+ Button::Left => 0x110, // BTN_LEFT+ Button::Right => 0x111, // BTN_RIGHT+ Button::Middle => 0x112, // BTN_MIDDLE+ }+ }+}++/// Commands the worker thread accepts. Each carries a reply channel so+/// the caller blocks until the EIS server has received the event.+enum Cmd {+ Click {+ x: f64,+ y: f64,+ button: Button,+ reply: Sender<anyhow::Result<()>>,+ },+ MoveAbsolute {+ x: f64,+ y: f64,+ reply: Sender<anyhow::Result<()>>,+ },+ Scroll {+ dx: f64,+ dy: f64,+ reply: Sender<anyhow::Result<()>>,+ },+ TypeText {+ text: String,+ reply: Sender<anyhow::Result<()>>,+ },+ PressKey {+ keycode: u32,+ reply: Sender<anyhow::Result<()>>,+ },+ Shutdown,+}++static TX: OnceLock<Sender<Cmd>> = OnceLock::new();++fn tx() -> anyhow::Result<&'static Sender<Cmd>> {+ TX.get().ok_or_else(|| anyhow::anyhow!("libei worker not started; call ensure_started() first"))+}++fn restore_token_path() -> Option<PathBuf> {+ let base = dirs::config_dir()?;+ Some(base.join("cua-driver").join("libei.token"))+}++fn read_restore_token() -> Option<String> {+ let path = restore_token_path()?;+ std::fs::read_to_string(path).ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty())+}++fn write_restore_token(token: &str) -> anyhow::Result<()> {+ let path = restore_token_path()+ .ok_or_else(|| anyhow::anyhow!("no config dir available for libei restore_token"))?;+ if let Some(parent) = path.parent() {+ std::fs::create_dir_all(parent).map_err(|e| {+ anyhow::anyhow!("failed to create {} for restore_token: {e}", parent.display())+ })?;+ }+ std::fs::write(&path, token)+ .map_err(|e| anyhow::anyhow!("failed to write libei restore_token to {}: {e}", path.display()))+}++/// Spawn the libei worker thread (idempotent — safe to call from every+/// MCP tool invocation; subsequent calls are no-ops).+pub fn ensure_started() -> anyhow::Result<()> {+ TX.get_or_init(|| {+ let (tx, rx) = bounded::<Cmd>(64);+ thread::Builder::new()+ .name("cua-libei-worker".into())+ .spawn(move || {+ if let Err(e) = worker(rx) {+ tracing::warn!("cua-libei-worker exited with error: {e}");+ }+ })+ .expect("spawn cua-libei-worker thread");+ tx+ });+ Ok(())+}++// ── Public API ───────────────────────────────────────────────────────────++/// Move the cursor to absolute (x, y) within the announced device region,+/// then press + release `button`. Blocks until the EIS server has+/// received the event sequence.+pub fn click(x: f64, y: f64, button: Button) -> anyhow::Result<()> {+ ensure_started()?;+ let (tx_r, rx_r) = bounded(1);+ tx()?.send(Cmd::Click { x, y, button, reply: tx_r })+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?+}++/// Move the cursor to absolute (x, y) inside the device region (no+/// button press).+pub fn move_absolute(x: f64, y: f64) -> anyhow::Result<()> {+ ensure_started()?;+ let (tx_r, rx_r) = bounded(1);+ tx()?.send(Cmd::MoveAbsolute { x, y, reply: tx_r })+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?+}++/// Scroll by (dx, dy) logical units. Positive y scrolls down.+pub fn scroll(dx: f64, dy: f64) -> anyhow::Result<()> {+ ensure_started()?;+ let (tx_r, rx_r) = bounded(1);+ tx()?.send(Cmd::Scroll { dx, dy, reply: tx_r })+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?+}++/// Inject a UTF-8 string via the `ei_text` interface (libei 1.6+).+/// Bypasses keycode reverse-mapping — works correctly for any layout.+pub fn type_text(text: &str) -> anyhow::Result<()> {+ ensure_started()?;+ let (tx_r, rx_r) = bounded(1);+ tx()?.send(Cmd::TypeText { text: text.to_string(), reply: tx_r })+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?+}++/// Press + release a key by evdev code (e.g. `KEY_ENTER` = 28). For+/// modifier combos use the keyboard interface's modifier state explicitly.+pub fn press_key(keycode: u32) -> anyhow::Result<()> {+ ensure_started()?;+ let (tx_r, rx_r) = bounded(1);+ tx()?.send(Cmd::PressKey { keycode, reply: tx_r })+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?+}++/// Cleanly stop the worker thread.+pub fn shutdown() {+ if let Some(tx) = TX.get() {+ let _ = tx.send(Cmd::Shutdown);+ }+}++// ── Worker thread ────────────────────────────────────────────────────────+//+// The worker owns the `reis::ei::Context`, the per-seat device map, and+// the negotiated input region. It runs a calloop event loop that handles+// both the EIS protocol and the inbound command channel.++fn worker(rx: Receiver<Cmd>) -> anyhow::Result<()> {+ // Phase 1 — acquire an EIS context.+ let context = open_eis_context()+ .map_err(|e| anyhow::anyhow!("failed to obtain EIS connection: {e}"))?;++ // Phase 2 — handshake. The reis API requires we call context.handshake()+ // and flush before the event loop starts polling.+ let _handshake = context.handshake();+ let _ = context.flush();++ // Phase 3 — run the calloop event loop. We use a calloop Generic source+ // wrapping the ei::Context's underlying socket so the loop wakes up on+ // EIS messages. The command channel is polled at the top of each loop+ // iteration.+ run_calloop(context, rx)+}++/// Open the EIS context. Fast path: $LIBEI_SOCKET (cua-compositor /+/// inject mode). Fallback: ashpd portal RemoteDesktop handshake.+fn open_eis_context() -> anyhow::Result<reis::ei::Context> {+ if let Some(ctx) = reis::ei::Context::connect_to_env()+ .map_err(|e| anyhow::anyhow!("env ei socket open failed: {e}"))?+ {+ return Ok(ctx);+ }++ // Portal RemoteDesktop fallback.+ use ashpd::desktop::{+ remote_desktop::{+ ConnectToEISOptions, DeviceType, RemoteDesktop, SelectDevicesOptions, StartOptions,+ },+ CreateSessionOptions, PersistMode,+ };+ use ashpd::enumflags2::BitFlags;+ use std::os::unix::net::UnixStream;++ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?;++ let fd = rt.block_on(async {+ let proxy = RemoteDesktop::new()+ .await+ .map_err(|e| anyhow::anyhow!("portal RemoteDesktop proxy unreachable: {e}. Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde."))?;+ let session = proxy+ .create_session(CreateSessionOptions::default())+ .await+ .map_err(|e| anyhow::anyhow!("portal create_session failed: {e}"))?;++ let mut select_opts = SelectDevicesOptions::default()+ .set_devices(BitFlags::<DeviceType>::from(DeviceType::Keyboard) | DeviceType::Pointer)+ .set_persist_mode(PersistMode::Application);+ if let Some(tok) = read_restore_token() {+ select_opts = select_opts.set_restore_token(Some(tok.as_str()));+ }++ proxy+ .select_devices(&session, select_opts)+ .await+ .map_err(|e| anyhow::anyhow!("portal select_devices failed: {e}"))?+ .response()+ .map_err(|e| anyhow::anyhow!("portal select_devices response error: {e}"))?;++ let started = proxy+ .start(&session, None, StartOptions::default())+ .await+ .map_err(|e| anyhow::anyhow!("portal start failed: {e}"))?+ .response()+ .map_err(|e| anyhow::anyhow!("portal start response error (user denied?): {e}"))?;++ // Best-effort persist of the restore_token. Without this, every+ // process restart re-prompts the user for consent — with it, ashpd+ // 0.13's PersistMode::Application + the stored token together+ // skip the dialog for the lifetime of the persistence grant+ // (typically per login session on GNOME, indefinite on KDE).+ if let Some(tok) = started.restore_token() {+ let _ = write_restore_token(tok);+ }++ let fd = proxy+ .connect_to_eis(&session, ConnectToEISOptions::default())+ .await+ .map_err(|e| anyhow::anyhow!("portal connect_to_eis failed: {e}"))?;+ anyhow::Ok(fd)+ })?;++ let stream = UnixStream::from(fd);+ let ctx = reis::ei::Context::new(stream)+ .map_err(|e| anyhow::anyhow!("reis ei::Context::new failed: {e}"))?;+ Ok(ctx)+}++// ── calloop dispatch ────────────────────────────────────────────────────+//+// The EIS protocol is event-driven and non-trivial — handshake → connection+// → seat → device → frame negotiation. We model the worker as a calloop+// EventLoop driving the ei::Context Generic source, with a separate+// channel source for inbound Cmd messages.+//+// For v1 we implement the handshake + seat-bind dance and provide+// click/move/type via the negotiated pointer/keyboard/text interfaces.+// Region selection picks the first announced ei_device::Region.++fn run_calloop(context: reis::ei::Context, rx: Receiver<Cmd>) -> anyhow::Result<()> {+ use calloop::{generic::Generic, EventLoop, Interest, Mode, PostAction};++ let mut event_loop: EventLoop<EisState> = EventLoop::try_new()+ .map_err(|e| anyhow::anyhow!("calloop EventLoop::try_new failed: {e}"))?;+ let handle = event_loop.handle();++ // EIS protocol source.+ let ctx_source = Generic::new(context, Interest::READ, Mode::Level);+ handle+ .insert_source(ctx_source, |_event, ctx, state: &mut EisState| {+ state.handle_eis_readable(unsafe { ctx.get_mut() })+ })+ .map_err(|e| anyhow::anyhow!("calloop insert_source(eis) failed: {e}"))?;++ let mut state = EisState::default();++ // Command channel: drain at the top of each loop iteration with a+ // bounded timeout so the EIS source stays responsive.+ loop {+ // Process pending commands.+ match rx.try_recv() {+ Ok(Cmd::Shutdown) => break,+ Ok(cmd) => state.handle_command(cmd),+ Err(crossbeam_channel::TryRecvError::Empty) => {}+ Err(crossbeam_channel::TryRecvError::Disconnected) => break,+ }++ event_loop+ .dispatch(Some(std::time::Duration::from_millis(20)), &mut state)+ .map_err(|e| anyhow::anyhow!("calloop dispatch failed: {e}"))?;++ // The handle_command path may have queued frame() requests; flush+ // them once per loop iteration so the EIS server sees them.+ if let Some(ctx) = state.context.as_ref() {+ let _ = ctx.flush();+ }+ }++ Ok(())+}++#[derive(Default)]+struct EisState {+ context: Option<reis::ei::Context>,+ seats: HashMap<reis::ei::Seat, SeatData>,+ devices: HashMap<reis::ei::Device, DeviceData>,+ sequence: u32,+ last_serial: u32,+}++#[derive(Default)]+struct SeatData {+ capabilities: HashMap<String, u64>,+}++/// One announced ei_device::Region — a single monitor's pixel rectangle+/// inside the seat's logical coordinate space. Multi-monitor sessions+/// announce one Region per output; we route each absolute (x, y) to the+/// region containing it so the cursor lands on the correct monitor.+#[derive(Clone, Copy, Debug)]+struct RegionRect {+ offset_x: f32,+ offset_y: f32,+ width: f32,+ height: f32,+}++impl RegionRect {+ fn contains(&self, x: f64, y: f64) -> bool {+ let xf = x as f32;+ let yf = y as f32;+ xf >= self.offset_x+ && xf < self.offset_x + self.width+ && yf >= self.offset_y+ && yf < self.offset_y + self.height+ }+}++#[derive(Default)]+struct DeviceData {+ device_type: Option<reis::ei::device::DeviceType>,+ /// Every Region the device announces between its creation and its+ /// first Done event. Empty until the EIS server has sent at least+ /// one Region.+ regions: Vec<RegionRect>,+ interfaces: HashMap<String, reis::Object>,+}++impl EisState {+ fn handle_eis_readable(+ &mut self,+ context: &mut reis::ei::Context,+ ) -> std::io::Result<calloop::PostAction> {+ use reis::PendingRequestResult;+ if self.context.is_none() {+ self.context = Some(context.clone());+ }+ if context.read().is_err() {+ return Ok(calloop::PostAction::Remove);+ }+ while let Some(result) = context.pending_event() {+ let request = match result {+ PendingRequestResult::Request(r) => r,+ _ => continue,+ };+ self.dispatch_eis_event(request);+ }+ let _ = context.flush();+ Ok(calloop::PostAction::Continue)+ }++ fn dispatch_eis_event(&mut self, event: reis::ei::Event) {+ use reis::ei::Event;+ match event {+ Event::Handshake(handshake, ev) => {+ if let reis::ei::handshake::Event::HandshakeVersion { .. } = ev {+ handshake.handshake_version(1);+ handshake.name("cua-driver");+ handshake.context_type(reis::ei::handshake::ContextType::Sender);+ for (iface, ver) in [+ ("ei_callback", 1u32),+ ("ei_connection", 1),+ ("ei_seat", 1),+ ("ei_device", 2),+ ("ei_pointer", 1),+ ("ei_pointer_absolute", 1),+ ("ei_button", 1),+ ("ei_scroll", 1),+ ("ei_keyboard", 1),+ ("ei_text", 1),+ ] {+ handshake.interface_version(iface, ver);+ }+ handshake.finish();+ }+ }+ Event::Connection(_conn, ev) => {+ if let reis::ei::connection::Event::Seat { seat } = ev {+ self.seats.insert(seat, SeatData::default());+ }+ }+ Event::Seat(seat, ev) => {+ let data = self.seats.entry(seat.clone()).or_default();+ match ev {+ reis::ei::seat::Event::Capability { mask, interface } => {+ data.capabilities.insert(interface, mask);+ }+ reis::ei::seat::Event::Done => {+ // Bind every input capability the seat advertises+ // (pointer / pointer_absolute / button / scroll /+ // keyboard / text). The server then announces+ // matching devices.+ let mut mask = 0u64;+ for k in [+ "ei_pointer",+ "ei_pointer_absolute",+ "ei_button",+ "ei_scroll",+ "ei_keyboard",+ "ei_text",+ ] {+ if let Some(m) = data.capabilities.get(k) {+ mask |= *m;+ }+ }+ if mask != 0 {+ seat.bind(mask);+ }+ }+ reis::ei::seat::Event::Device { device } => {+ self.devices.insert(device, DeviceData::default());+ }+ _ => {}+ }+ }+ Event::Device(device, ev) => {+ let data = self.devices.entry(device.clone()).or_default();+ match ev {+ reis::ei::device::Event::DeviceType { device_type } => {+ data.device_type = Some(device_type);+ }+ reis::ei::device::Event::Interface { object } => {+ data.interfaces.insert(object.interface().to_owned(), object);+ }+ reis::ei::device::Event::Region { offset_x, offset_y, width, hight, scale: _ } => {+ // reis 0.7 keeps the misspelled "hight" field name+ // for ABI compatibility — see the protocol comment.+ if width > 0 && hight > 0 {+ data.regions.push(RegionRect {+ offset_x: offset_x as f32,+ offset_y: offset_y as f32,+ width: width as f32,+ height: hight as f32,+ });+ }+ }+ reis::ei::device::Event::Resumed { serial } => {+ self.last_serial = serial;+ }+ _ => {}+ }+ }+ _ => {}+ }+ }++ fn handle_command(&mut self, cmd: Cmd) {+ let result = self.run_command(&cmd);+ // Send the reply on whichever channel this command carries.+ match cmd {+ Cmd::Click { reply, .. } => { let _ = reply.send(result); }+ Cmd::MoveAbsolute { reply, .. } => { let _ = reply.send(result); }+ Cmd::Scroll { reply, .. } => { let _ = reply.send(result); }+ Cmd::TypeText { reply, .. } => { let _ = reply.send(result); }+ Cmd::PressKey { reply, .. } => { let _ = reply.send(result); }+ Cmd::Shutdown => {}+ }+ }++ fn run_command(&mut self, cmd: &Cmd) -> anyhow::Result<()> {+ match cmd {+ Cmd::Click { x, y, button, .. } => {+ let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?;+ if let Some(ptr_abs) = device_interface::<reis::ei::PointerAbsolute>(&self.devices, &device) {+ device.start_emulating(self.sequence, self.last_serial);+ self.sequence = self.sequence.wrapping_add(1);+ ptr_abs.motion_absolute(rel_x, rel_y);+ }+ if let Some(btn) = device_interface::<reis::ei::Button>(&self.devices, &device) {+ btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press);+ device.frame(self.last_serial, 0);+ btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released);+ device.frame(self.last_serial, 1);+ }+ device.stop_emulating(self.last_serial);+ }+ Cmd::MoveAbsolute { x, y, .. } => {+ let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?;+ if let Some(ptr_abs) = device_interface::<reis::ei::PointerAbsolute>(&self.devices, &device) {+ device.start_emulating(self.sequence, self.last_serial);+ self.sequence = self.sequence.wrapping_add(1);+ ptr_abs.motion_absolute(rel_x, rel_y);+ device.frame(self.last_serial, 0);+ device.stop_emulating(self.last_serial);+ }+ }+ Cmd::Scroll { dx, dy, .. } => {+ let device = self.any_pointer_device()+ .ok_or_else(|| anyhow::anyhow!("no EIS pointer device negotiated yet — wait for handshake"))?;+ if let Some(scroll) = device_interface::<reis::ei::Scroll>(&self.devices, &device) {+ device.start_emulating(self.sequence, self.last_serial);+ self.sequence = self.sequence.wrapping_add(1);+ scroll.scroll(*dx as f32, *dy as f32);+ device.frame(self.last_serial, 0);+ device.stop_emulating(self.last_serial);+ }+ }+ Cmd::TypeText { text, .. } => {+ let device = self.any_pointer_device()+ .ok_or_else(|| anyhow::anyhow!("no EIS device negotiated yet — wait for handshake"))?;+ if let Some(text_iface) = device_interface::<reis::ei::Text>(&self.devices, &device) {+ device.start_emulating(self.sequence, self.last_serial);+ self.sequence = self.sequence.wrapping_add(1);+ text_iface.utf8(text);+ device.frame(self.last_serial, 0);+ device.stop_emulating(self.last_serial);+ }+ }+ Cmd::PressKey { keycode, .. } => {+ let device = self.any_pointer_device()+ .ok_or_else(|| anyhow::anyhow!("no EIS device negotiated yet — wait for handshake"))?;+ if let Some(kb) = device_interface::<reis::ei::Keyboard>(&self.devices, &device) {+ device.start_emulating(self.sequence, self.last_serial);+ self.sequence = self.sequence.wrapping_add(1);+ kb.key(*keycode, reis::ei::keyboard::KeyState::Press);+ device.frame(self.last_serial, 0);+ kb.key(*keycode, reis::ei::keyboard::KeyState::Released);+ device.frame(self.last_serial, 1);+ device.stop_emulating(self.last_serial);+ }+ }+ Cmd::Shutdown => {}+ }+ if let Some(ctx) = self.context.as_ref() {+ let _ = ctx.flush();+ }+ Ok(())+ }++ /// Pick a pointer device whose announced Region contains `(x, y)`,+ /// and translate the absolute screen coordinates into the device's+ /// region-local pixel coordinates.+ ///+ /// Multi-monitor seats announce one Region per output; routing each+ /// (x, y) by containment is what lets a click on monitor #2 actually+ /// land on monitor #2. When no device's region contains the point —+ /// e.g. coords just past the edge of an output, or a session with+ /// a single device that hasn't announced any regions yet — we fall+ /// back to the first pointer device with the raw coordinates, which+ /// matches the pre-multi-monitor behaviour.+ fn pointer_device_for(&self, x: f64, y: f64) -> anyhow::Result<(reis::ei::Device, f32, f32)> {+ let is_pointer = |data: &DeviceData| matches!(+ data.device_type,+ Some(reis::ei::device::DeviceType::Virtual) | Some(reis::ei::device::DeviceType::Physical)+ );+ for (device, data) in self.devices.iter() {+ if !is_pointer(data) {+ continue;+ }+ for region in &data.regions {+ if region.contains(x, y) {+ let rel_x = x as f32 - region.offset_x;+ let rel_y = y as f32 - region.offset_y;+ return Ok((device.clone(), rel_x, rel_y));+ }+ }+ }+ // Fallback: first pointer device, raw coords (single-region behaviour).+ let device = self.any_pointer_device()+ .ok_or_else(|| anyhow::anyhow!("no EIS pointer device negotiated yet — wait for handshake"))?;+ Ok((device, x as f32, y as f32))+ }++ fn any_pointer_device(&self) -> Option<reis::ei::Device> {+ self.devices.iter()+ .find(|(_, data)| matches!(+ data.device_type,+ Some(reis::ei::device::DeviceType::Virtual) | Some(reis::ei::device::DeviceType::Physical)+ ))+ .map(|(d, _)| d.clone())+ }+}++fn device_interface<T: reis::Interface>(+ devices: &HashMap<reis::ei::Device, DeviceData>,+ device: &reis::ei::Device,+) -> Option<T> {+ devices.get(device)?+ .interfaces+ .get(T::NAME)?+ .clone()+ .downcast()+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rsindex 5c7d0bf9ad..da9cf1457d 100644--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs@@ -1,16 +1,36 @@//! Native-Wayland backend.//!//! Used when running under a Wayland compositor with no X11 (WAYLAND_DISPLAY-//! set, DISPLAY unset). The first slice enumerates toplevels via-//! `zwlr_foreign_toplevel_manager_v1` (wlroots: labwc, sway, …) so-//! `list_windows` works on native Wayland. Capture and input slices land next.+//! set, DISPLAY unset). Enumerates toplevels via+//! `zwlr_foreign_toplevel_manager_v1`, captures per-output screenshots via+//! `zwlr_screencopy_manager_v1` + `wl_shm` (native — `grim` remains a+//! fallback), and synthesises pointer / scroll / drag input via+//! `zwlr_virtual_pointer_v1`. Per-window image capture is deferred until+//! `ext-foreign-toplevel-image-capture-source-v1` lands in+//! `wayland-protocols-wlr`; until then `screenshot_window_dispatch` returns a+//! typed error on pure Wayland.++pub mod persistent_vptr;+pub mod portal_screenshot;+pub mod overlay;+pub mod ext_screencopy;+pub mod portal_screencast;+pub mod libei;use std::collections::HashMap;use wayland_client::{event_created_child,- protocol::{wl_output::{self, WlOutput}, wl_pointer::ButtonState, wl_registry, wl_seat::WlSeat},- Connection, Dispatch, Proxy, QueueHandle,+ protocol::{+ wl_buffer::WlBuffer,+ wl_output::{self, WlOutput},+ wl_pointer::{Axis, AxisSource, ButtonState},+ wl_registry,+ wl_seat::WlSeat,+ wl_shm::{self, WlShm},+ wl_shm_pool::WlShmPool,+ },+ Connection, Dispatch, Proxy, QueueHandle, WEnum,};use wayland_protocols_wlr::foreign_toplevel::v1::client::{zwlr_foreign_toplevel_handle_v1::{self as ftl_handle, ZwlrForeignToplevelHandleV1},@@ -18,6 +38,10 @@ use wayland_protocols_wlr::foreign_toplevel::v1::client::{self as ftl_manager, ZwlrForeignToplevelManagerV1, EVT_TOPLEVEL_OPCODE,},};+use wayland_protocols_wlr::screencopy::v1::client::{+ zwlr_screencopy_frame_v1::{self as scrcopy_frame, ZwlrScreencopyFrameV1},+ zwlr_screencopy_manager_v1::ZwlrScreencopyManagerV1,+};use wayland_protocols_wlr::virtual_pointer::v1::client::{zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,@@ -34,12 +58,14 @@ pub const ENABLE_WAYLAND_ENV: &str = "CUA_DRIVER_RS_ENABLE_WAYLAND";/// Whether the user has opted into the experimental native-Wayland backend.///-/// The Wayland backend is incomplete (toplevel enumeration + virtual-pointer /-/// virtual-keyboard input via the wlroots protocols; capture and AT-SPI parity-/// are still landing), so it stays OFF by default and a pure-Wayland session is-/// reported as unsupported unless the user explicitly sets-/// `CUA_DRIVER_RS_ENABLE_WAYLAND=1`. Any value other than empty / `0` / `false`-/// enables it.+/// The Wayland backend covers toplevel enumeration, per-output capture+/// (native screencopy + a `grim` fallback), and virtual-pointer /+/// virtual-keyboard input via the wlroots protocols. Per-window image+/// capture still depends on the staging `ext-image-copy-capture-v1`+/// protocol, so the backend stays OFF by default and a pure-Wayland+/// session is reported as unsupported unless the user explicitly sets+/// `CUA_DRIVER_RS_ENABLE_WAYLAND=1`. Any value other than empty / `0` /+/// `false` enables it.pub fn wayland_enabled() -> bool {match std::env::var(ENABLE_WAYLAND_ENV) {Ok(v) => {@@ -133,6 +159,19 @@ struct Toplevel {closed: bool,}+/// Per-capture in-flight state populated by the screencopy frame Dispatch.+#[derive(Default)]+struct CaptureState {+ /// wl_shm format code (Argb8888 / Xrgb8888 / …).+ format: Option<u32>,+ width: u32,+ height: u32,+ stride: u32,+ y_invert: bool,+ ready: bool,+ failed: bool,+}+#[derive(Default)]struct State {manager: Option<ZwlrForeignToplevelManagerV1>,@@ -144,8 +183,13 @@ struct State {// Virtual-pointer manager + output dimensions, so `click` can land a real// button press at the output centre (over the just-activated window).vptr_manager: Option<ZwlrVirtualPointerManagerV1>,+ output: Option<WlOutput>,output_w: u32,output_h: u32,+ // Native screencopy capture state.+ scrcopy_manager: Option<ZwlrScreencopyManagerV1>,+ shm: Option<WlShm>,+ capture: CaptureState,}impl Dispatch<wl_registry::WlRegistry, ()> for State {@@ -168,7 +212,15 @@ impl Dispatch<wl_registry::WlRegistry, ()> for State {state.vptr_manager =Some(registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(name, version.min(2), qh, ()));} else if interface == WlOutput::interface().name {- registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ());+ let out = registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ());+ if state.output.is_none() {+ state.output = Some(out);+ }+ } else if interface == ZwlrScreencopyManagerV1::interface().name {+ state.scrcopy_manager =+ Some(registry.bind::<ZwlrScreencopyManagerV1, _, _>(name, version.min(3), qh, ()));+ } else if interface == WlShm::interface().name {+ state.shm = Some(registry.bind::<WlShm, _, _>(name, version.min(1), qh, ()));}}}@@ -229,6 +281,91 @@ impl Dispatch<ZwlrVirtualPointerV1, ()> for State {}}+impl Dispatch<WlShm, ()> for State {+ fn event(+ _: &mut Self,+ _: &WlShm,+ _: wl_shm::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ // wl_shm advertises supported formats via `format` events; we don't+ // need to track them — screencopy tells us exactly which format to use+ // for the frame buffer.+ }+}++impl Dispatch<WlShmPool, ()> for State {+ fn event(+ _: &mut Self,+ _: &WlShmPool,+ _: wayland_client::protocol::wl_shm_pool::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ }+}++impl Dispatch<WlBuffer, ()> for State {+ fn event(+ _: &mut Self,+ _: &WlBuffer,+ _: wayland_client::protocol::wl_buffer::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ }+}++impl Dispatch<ZwlrScreencopyManagerV1, ()> for State {+ fn event(+ _: &mut Self,+ _: &ZwlrScreencopyManagerV1,+ _: <ZwlrScreencopyManagerV1 as Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ }+}++impl Dispatch<ZwlrScreencopyFrameV1, ()> for State {+ fn event(+ state: &mut Self,+ _: &ZwlrScreencopyFrameV1,+ event: scrcopy_frame::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ match event {+ scrcopy_frame::Event::Buffer { format, width, height, stride } => {+ if let WEnum::Value(fmt) = format {+ state.capture.format = Some(fmt as u32);+ }+ state.capture.width = width;+ state.capture.height = height;+ state.capture.stride = stride;+ }+ scrcopy_frame::Event::Flags { flags } => {+ if let WEnum::Value(f) = flags {+ state.capture.y_invert = f.contains(scrcopy_frame::Flags::YInvert);+ }+ }+ scrcopy_frame::Event::Ready { .. } => {+ state.capture.ready = true;+ }+ scrcopy_frame::Event::Failed => {+ state.capture.failed = true;+ }+ _ => {}+ }+ }+}+impl Dispatch<ZwlrForeignToplevelManagerV1, ()> for State {fn event(_state: &mut Self,@@ -305,12 +442,27 @@ pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> {Ok(out)}-/// Capture the Wayland output as PNG bytes via `grim` (the wlroots screenshot-/// tool — wlr-screencopy under the hood, works on labwc/sway). This mirrors the-/// X11 backend shelling out to `import`/`xwd`. foreign-toplevel exposes no-/// per-window geometry, so this captures the whole output; that is sufficient-/// for `get_window_state`'s vision payload.+// ── Capture (native screencopy + grim fallback) ──────────────────────────────++/// Capture the Wayland output as PNG bytes via `zwlr_screencopy_manager_v1`.+///+/// Binds the screencopy manager plus `wl_shm`, asks the compositor to copy the+/// next frame of the first advertised output into a shm buffer, channel-swaps+/// from the compositor's pixel format to RGBA, and encodes a PNG via the+/// existing `image` crate. Falls back to shelling out to `grim` when the+/// screencopy manager or `wl_shm` is unavailable so users on lighter wlroots+/// builds stay supported.pub fn screenshot_bytes() -> anyhow::Result<Vec<u8>> {+ match capture_via_screencopy() {+ Ok(bytes) => return Ok(bytes),+ Err(e) => tracing::warn!("native screencopy failed, falling back to grim: {e}"),+ }+ capture_via_grim()+}++/// Shell out to `grim -t png -` — the wlroots reference screenshot tool. Kept+/// as the last-resort fallback for compositors that hide screencopy.+fn capture_via_grim() -> anyhow::Result<Vec<u8>> {let out = std::process::Command::new("grim").args(["-t", "png", "-"]).output()?;if !out.status.success() {anyhow::bail!("grim failed: {}", String::from_utf8_lossy(&out.stderr));@@ -321,7 +473,202 @@ pub fn screenshot_bytes() -> anyhow::Result<Vec<u8>> {Ok(out.stdout)}-/// Capture dispatcher: native Wayland (grim) when applicable, else X11.+/// Native screencopy path: bind manager + shm, allocate an anon mmap buffer,+/// request a copy, wait for Ready, swap channels, encode PNG. Returns an error+/// if any global is missing or the compositor flags the capture as failed.+fn capture_via_screencopy() -> anyhow::Result<Vec<u8>> {+ let conn = Connection::connect_to_env()?;+ let mut queue = conn.new_event_queue::<State>();+ let qh = queue.handle();+ conn.display().get_registry(&qh, ());++ let mut state = State::default();+ queue.roundtrip(&mut state)?;+ queue.roundtrip(&mut state)?; // outputs report their Mode++ let manager = state+ .scrcopy_manager+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose zwlr_screencopy_manager_v1"))?;+ let shm = state+ .shm+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_shm"))?;+ let output = state+ .output+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_output to capture"))?;++ let frame = manager.capture_output(0, &output, &qh, ());+ // Drain Buffer / Flags events; spin until Ready or Failed (or timeout).+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);+ let mut buffer: Option<WlBuffer> = None;+ let mut pool: Option<WlShmPool> = None;+ let mut mmap_ptr: *mut libc::c_void = std::ptr::null_mut();+ let mut mmap_len: usize = 0;+ let mut fd: i32 = -1;++ loop {+ queue.roundtrip(&mut state)?;+ if state.capture.failed {+ anyhow::bail!("compositor signalled screencopy failure");+ }+ if state.capture.ready {+ break;+ }+ // Once we know the buffer params, allocate + send copy exactly once.+ if buffer.is_none()+ && state.capture.format.is_some()+ && state.capture.stride > 0+ && state.capture.height > 0+ {+ let size = (state.capture.stride as usize)+ .checked_mul(state.capture.height as usize)+ .ok_or_else(|| anyhow::anyhow!("screencopy buffer size overflow"))?;+ let (anon_fd, p) = anon_shm(size)?;+ fd = anon_fd;+ mmap_ptr = p;+ mmap_len = size;+ use std::os::fd::AsFd as _;+ let pool_fd = unsafe { borrowed_fd(fd) };+ let p = shm.create_pool(pool_fd.as_fd(), size as i32, &qh, ());+ let fmt_raw = state.capture.format.unwrap();+ let fmt: wl_shm::Format = match wl_shm::Format::try_from(fmt_raw) {+ Ok(f) => f,+ Err(_) => {+ cleanup_mmap(mmap_ptr, mmap_len, fd);+ anyhow::bail!("compositor advertised unsupported wl_shm format {fmt_raw:#x}");+ }+ };+ let b = p.create_buffer(+ 0,+ state.capture.width as i32,+ state.capture.height as i32,+ state.capture.stride as i32,+ fmt,+ &qh,+ (),+ );+ frame.copy(&b);+ buffer = Some(b);+ pool = Some(p);+ }+ if std::time::Instant::now() >= deadline {+ cleanup_mmap(mmap_ptr, mmap_len, fd);+ anyhow::bail!("screencopy timed out waiting for frame");+ }+ }++ let result = (|| -> anyhow::Result<Vec<u8>> {+ let w = state.capture.width;+ let h = state.capture.height;+ let stride = state.capture.stride as usize;+ let format = state.capture.format.unwrap_or(0);+ if mmap_ptr.is_null() || mmap_len == 0 {+ anyhow::bail!("screencopy ready without a backing buffer");+ }+ let raw = unsafe { std::slice::from_raw_parts(mmap_ptr as *const u8, mmap_len) };+ let mut rgba = Vec::with_capacity((w as usize) * (h as usize) * 4);+ for row in 0..(h as usize) {+ let src_row = if state.capture.y_invert { (h as usize) - 1 - row } else { row };+ let base = src_row * stride;+ for col in 0..(w as usize) {+ let px = &raw[base + col * 4..base + col * 4 + 4];+ let (r, g, b, a) = match wl_shm::Format::try_from(format).ok() {+ // Argb8888 / Xrgb8888 over wl_shm are little-endian BGRA / BGRX.+ Some(wl_shm::Format::Argb8888) => (px[2], px[1], px[0], px[3]),+ Some(wl_shm::Format::Xrgb8888) => (px[2], px[1], px[0], 255),+ Some(wl_shm::Format::Abgr8888) => (px[0], px[1], px[2], px[3]),+ Some(wl_shm::Format::Xbgr8888) => (px[0], px[1], px[2], 255),+ _ => (px[2], px[1], px[0], px[3]),+ };+ rgba.extend_from_slice(&[r, g, b, a]);+ }+ }+ cua_driver_core::image_utils::encode_rgba_to_png(&rgba, w, h)+ })();++ // Always tear down regardless of result.+ if let Some(b) = buffer {+ b.destroy();+ }+ if let Some(p) = pool {+ p.destroy();+ }+ frame.destroy();+ let _ = queue.roundtrip(&mut state);+ cleanup_mmap(mmap_ptr, mmap_len, fd);++ result+}++/// Allocate an anonymous shared-memory file of `size` bytes and mmap it RW.+/// Returns the raw fd and the mmap pointer; the caller is responsible for+/// passing both to [`cleanup_mmap`] when done.+pub(crate) fn anon_shm(size: usize) -> anyhow::Result<(i32, *mut libc::c_void)> {+ // memfd_create is Linux-only and is the cleanest path; fall back to+ // shm_open if memfd isn't available for any reason.+ let name = b"cua-scrcopy\0";+ let fd = unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) };+ if fd < 0 {+ return Err(anyhow::anyhow!("memfd_create failed: {}", std::io::Error::last_os_error()));+ }+ let rc = unsafe { libc::ftruncate(fd, size as libc::off_t) };+ if rc != 0 {+ let err = std::io::Error::last_os_error();+ unsafe { libc::close(fd) };+ return Err(anyhow::anyhow!("ftruncate failed: {err}"));+ }+ let p = unsafe {+ libc::mmap(+ std::ptr::null_mut(),+ size,+ libc::PROT_READ | libc::PROT_WRITE,+ libc::MAP_SHARED,+ fd,+ 0,+ )+ };+ if p == libc::MAP_FAILED {+ let err = std::io::Error::last_os_error();+ unsafe { libc::close(fd) };+ return Err(anyhow::anyhow!("mmap failed: {err}"));+ }+ Ok((fd, p))+}++/// Unmap and close the screencopy backing buffer; safe to call with the+/// sentinel values left from a never-allocated buffer.+pub(crate) fn cleanup_mmap(ptr: *mut libc::c_void, len: usize, fd: i32) {+ if !ptr.is_null() && len > 0 {+ unsafe { libc::munmap(ptr, len) };+ }+ if fd >= 0 {+ unsafe { libc::close(fd) };+ }+}++/// Borrow a raw fd as an `OwnedFd` for wl_shm.create_pool. The pool keeps+/// its own reference; we close our copy via [`cleanup_mmap`].+///+/// SAFETY: caller must guarantee `fd` is a valid open file descriptor.+/// `libc::dup` may fail (returning -1, errno set), in which case we panic+/// instead of constructing an `OwnedFd` from -1 (which would have UB on+/// drop). Callers that need fallible behaviour should use the dup syscall+/// directly and check the result before wrapping.+pub(crate) unsafe fn borrowed_fd(fd: i32) -> std::os::fd::OwnedFd {+ use std::os::fd::FromRawFd;+ let dup = libc::dup(fd);+ if dup < 0 {+ let err = std::io::Error::last_os_error();+ panic!("dup({fd}) failed: {err}");+ }+ std::os::fd::OwnedFd::from_raw_fd(dup)+}++/// Capture dispatcher: native Wayland (screencopy with grim fallback) when+/// applicable, else X11. Mirrors `screenshot_window_dispatch` for the+/// output-level path used by `get_window_state`'s vision payload.pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {if is_wayland() {screenshot_bytes()@@ -330,61 +677,302 @@ pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {}}-/// Click a native Wayland toplevel identified by its `window_id` (the-/// foreign-toplevel protocol id from `list_windows`). Wayland forbids a client-/// from knowing another window's on-screen geometry, so we cannot map the-/// caller's window-local x/y to a global pointer position. Instead, the-/// focused-input model is two steps:-/// 1. `activate` the target toplevel (foreign-toplevel) — focus+raise it, so-/// on a stacking layout it fills the output.-/// 2. land a real virtual-pointer button press at the output centre (now over-/// the target). This both visually clicks and — crucially on sway, whose-/// headless seat has no keyboard until a pointer interaction occurs — makes-/// the compositor route the subsequent virtual-keyboard input to the-/// target. (labwc routes it from `activate` alone; the centre click is-/// harmless there since the activated window is centred under the cursor.)-pub fn click(window_id: u64) -> anyhow::Result<()> {+/// Display-level capture dispatcher. Cascade:+/// 1. Native Wayland on wlroots: zwlr_screencopy_manager_v1 (fast, zero+/// consent).+/// 2. Wayland but no wlroots screencopy globals (GNOME/KDE/COSMIC):+/// xdg-desktop-portal Screenshot via ashpd. Triggers consent prompt+/// on first use per session.+/// 3. X11: existing root-window path.+pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {+ if is_wayland() {+ // Tier 1: native wlroots screencopy (fast, zero consent).+ match screenshot_bytes() {+ Ok(bytes) => return Ok(bytes),+ Err(e) => {+ tracing::debug!(+ "wlroots screencopy unavailable ({e}); trying ext-image-copy-capture-v1"+ );+ }+ }+ // Tier 2: ext-image-copy-capture-v1 (sway 1.10+, labwc 0.8+, niri,+ // hyprland, KDE 6.2+, GNOME 47+).+ match ext_screencopy::screenshot_via_ext_copy() {+ Ok(bytes) => return Ok(bytes),+ Err(e) => {+ tracing::debug!(+ "ext-image-copy-capture-v1 unavailable ({e}); trying xdg-desktop-portal"+ );+ }+ }+ // Tier 3: xdg-desktop-portal (GNOME, KDE, COSMIC fallback).+ match portal_screenshot::screenshot_via_portal() {+ Ok(bytes) => return Ok(bytes),+ Err(e) => {+ tracing::debug!(+ "xdg-desktop-portal Screenshot unavailable ({e}); falling through to X11"+ );+ }+ }+ }+ // Final fallback: X11 root window. Call the X11-only path explicitly+ // so we don't re-enter screenshot_display_bytes (which routes back here+ // on Wayland — would loop forever).+ crate::capture::screenshot_display_bytes_x11()+}++/// Per-window capture dispatcher. On X11 forwards to the existing window+/// capture path; on pure Wayland returns a typed error pointing at the+/// staging `ext-image-copy-capture-v1` protocol — wlr-screencopy is+/// output-only, and `foreign-toplevel` exposes no per-window geometry to+/// crop with.+pub fn screenshot_window_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {+ if is_wayland() {+ anyhow::bail!(+ "per-window screenshot is not yet supported on native Wayland — \+ zwlr_screencopy_manager_v1 is output-only and ext-image-copy-capture-v1 \+ is not yet shipped in wayland-protocols-wlr. Run under XWayland to crop \+ to a single window, or capture the full output instead."+ );+ }+ crate::capture::screenshot_window_bytes(xid)+}++// ── Input session helper ─────────────────────────────────────────────────────++/// Live virtual-pointer session: connection + queue + the bound objects every+/// pointer op (click, scroll, drag) needs. Returned by [`open_vptr_session`].+pub struct VptrSession {+ pub conn: Connection,+ pub queue: wayland_client::EventQueue<State>,+ pub state: State,+ pub seat: WlSeat,+ pub vptr: ZwlrVirtualPointerV1,+ pub output_w: u32,+ pub output_h: u32,+}++/// Bind manager + seat + virtual-pointer + first output, optionally activate a+/// foreign-toplevel by `window_id` so the synthesised events land on it, and+/// return the live session that scroll / drag / click reuse. Wayland forbids a+/// client from knowing another window's on-screen geometry, so we drive every+/// pointer event in *output* coordinates and rely on the activated toplevel+/// covering the centre.+pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<VptrSession> {let conn = Connection::connect_to_env()?;let mut queue = conn.new_event_queue::<State>();let qh = queue.handle();conn.display().get_registry(&qh, ());let mut state = State::default();- queue.roundtrip(&mut state)?; // bind manager + seat + vptr + outputs+ queue.roundtrip(&mut state)?;if state.manager.is_none() {anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1");}for _ in 0..4 {- queue.roundtrip(&mut state)?; // drain toplevel/handle creation + output mode+ queue.roundtrip(&mut state)?;}- let id = window_id as u32;- let handle = state- .handles- .get(&id)- .ok_or_else(|| anyhow::anyhow!("no native Wayland toplevel for window_id {window_id}"))?- .clone();let seat = state.seat.clone()- .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_seat to activate the window"))?;- handle.activate(&seat);- queue.roundtrip(&mut state)?; // flush activate so the target is focused/raised-- // Land a button press at the output centre (over the now-focused window).- if let Some(mgr) = state.vptr_manager.clone() {- let (w, h) = (state.output_w.max(1), state.output_h.max(1));- let vptr = mgr.create_virtual_pointer(Some(&seat), &qh, ());- vptr.motion_absolute(0, w / 2, h / 2, w, h);- vptr.frame();- vptr.button(0, BTN_LEFT, ButtonState::Pressed);- vptr.frame();- vptr.button(0, BTN_LEFT, ButtonState::Released);- vptr.frame();- queue.roundtrip(&mut state)?;- vptr.destroy();+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_seat for virtual-pointer input"))?;+ let mgr = state+ .vptr_manager+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose zwlr_virtual_pointer_manager_v1"))?;++ if let Some(id) = activate_window_id {+ let handle = state+ .handles+ .get(&id)+ .cloned()+ .ok_or_else(|| anyhow::anyhow!("no native Wayland toplevel for window_id {id}"))?;+ handle.activate(&seat);queue.roundtrip(&mut state)?;}++ let vptr = mgr.create_virtual_pointer(Some(&seat), &qh, ());+ let (output_w, output_h) = (state.output_w.max(1), state.output_h.max(1));+ Ok(VptrSession { conn, queue, state, seat, vptr, output_w, output_h })+}++/// Map a cua/X11 pointer button (1=left / 2=middle / 3=right) to its evdev+/// code, which is what `zwlr_virtual_pointer_v1::button` expects.+pub fn evdev_pointer_button(button: u8) -> u32 {+ match button {+ 2 => 0x112, // BTN_MIDDLE+ 3 => 0x111, // BTN_RIGHT+ _ => 0x110, // BTN_LEFT+ }+}++/// Click a native Wayland toplevel identified by its `window_id` (the+/// foreign-toplevel protocol id from `list_windows`) at output-relative+/// `(x, y)`, with `button` (1/2/3 = left/middle/right) emitted `count` times.+/// Coordinates default to the output centre when both x and y are zero so+/// the legacy focus-based behaviour is preserved when callers can't supply+/// real coords. A short delay between iterations gives the compositor time+/// to discriminate single vs. double clicks.+pub fn click(window_id: u64, x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> {+ let mut sess = open_vptr_session(Some(window_id as u32))?;+ let (w, h) = (sess.output_w, sess.output_h);+ let (px, py) = if x == 0 && y == 0 {+ ((w / 2) as i32, (h / 2) as i32)+ } else {+ (x, y)+ };+ let px = px.clamp(0, w as i32 - 1) as u32;+ let py = py.clamp(0, h as i32 - 1) as u32;+ let btn = evdev_pointer_button(button);+ for i in 0..count.max(1) {+ if i > 0 {+ std::thread::sleep(std::time::Duration::from_millis(80));+ }+ sess.vptr.motion_absolute(0, px, py, w, h);+ sess.vptr.frame();+ sess.vptr.button(0, btn, ButtonState::Pressed);+ sess.vptr.frame();+ sess.vptr.button(0, btn, ButtonState::Released);+ sess.vptr.frame();+ sess.queue.roundtrip(&mut sess.state)?;+ }+ // Keep the synthetic-cursor registry in sync with the warp we just+ // performed so a subsequent `get_cursor_position` reflects reality.+ record_synth_cursor(px as i32, py as i32);+ sess.vptr.destroy();+ sess.queue.roundtrip(&mut sess.state)?;+ Ok(())+}++/// Synthesize a vertical or horizontal scroll on the activated toplevel. Each+/// tick emits an `axis_source(wheel)` + `axis_discrete(1)` pair through the+/// virtual-pointer protocol, mirroring how a real wheel notch decomposes. The+/// magnitude follows wl_pointer convention: ±10 (in wl_fixed = ×256) per tick.+pub fn scroll(window_id: u64, direction: &str, amount: u32) -> anyhow::Result<()> {+ let mut sess = open_vptr_session(Some(window_id as u32))?;+ let (axis, sign): (Axis, i32) = match direction.to_ascii_lowercase().as_str() {+ "up" => (Axis::VerticalScroll, -1),+ "down" => (Axis::VerticalScroll, 1),+ "left" => (Axis::HorizontalScroll, -1),+ "right" => (Axis::HorizontalScroll, 1),+ other => anyhow::bail!("unknown scroll direction: {other}"),+ };+ // axis_discrete: `value` is logical units (the wayland-rs wrapper+ // converts to wl_fixed internally); `discrete` is the tick count.+ let value: f64 = (sign as f64) * 10.0;+ for i in 0..amount.max(1) {+ if i > 0 {+ std::thread::sleep(std::time::Duration::from_millis(25));+ }+ sess.vptr.axis_source(AxisSource::Wheel);+ sess.vptr.axis_discrete(0, axis, value, sign);+ sess.vptr.frame();+ sess.queue.roundtrip(&mut sess.state)?;+ }+ sess.vptr.destroy();+ sess.queue.roundtrip(&mut sess.state)?;+ Ok(())+}++/// Last cursor position the agent warped to via `move_cursor_absolute`.+/// Wayland exposes no protocol for clients to read the real global cursor+/// position; a Wayland-conformant `get_cursor_position` can therefore only+/// report what THIS process synthesized. Updated every `motion_absolute`+/// emitted from `move_cursor_absolute` / `click` / `drag`.+static SYNTH_CURSOR_POS: std::sync::OnceLock<std::sync::Mutex<Option<(i32, i32)>>> =+ std::sync::OnceLock::new();++fn record_synth_cursor(x: i32, y: i32) {+ let cell = SYNTH_CURSOR_POS.get_or_init(|| std::sync::Mutex::new(None));+ if let Ok(mut g) = cell.lock() {+ *g = Some((x, y));+ }+}++/// Returns the last `(x, y)` this process warped the cursor to via the+/// Wayland virtual-pointer protocol, or `None` if no warp has happened in+/// this process. The reading is "synthetic": Wayland forbids clients from+/// querying the real cursor position, so this value diverges from reality+/// the moment the user moves their physical mouse. Callers should surface+/// `source: "synthetic"` in the structured payload.+pub fn last_synth_cursor_pos() -> Option<(i32, i32)> {+ SYNTH_CURSOR_POS.get_or_init(|| std::sync::Mutex::new(None))+ .lock()+ .ok()+ .and_then(|g| *g)+}++/// Warp the cursor to absolute output coordinates `(x, y)` using+/// `zwlr_virtual_pointer_v1::motion_absolute`. Clamps to the output bounds+/// reported by `open_vptr_session`. Emits a motion + frame and roundtrips so+/// the compositor commits the warp before returning. Records the position in+/// the synthetic-cursor registry so `last_synth_cursor_pos` can report it.+pub fn move_cursor_absolute(window_id: Option<u64>, x: i32, y: i32) -> anyhow::Result<()> {+ let mut sess = open_vptr_session(window_id.map(|w| w as u32))?;+ let (w, h) = (sess.output_w, sess.output_h);+ let px = x.clamp(0, (w as i32).saturating_sub(1)) as u32;+ let py = y.clamp(0, (h as i32).saturating_sub(1)) as u32;+ sess.vptr.motion_absolute(0, px, py, w, h);+ sess.vptr.frame();+ sess.queue.roundtrip(&mut sess.state)?;+ record_synth_cursor(px as i32, py as i32);+ sess.vptr.destroy();+ sess.queue.roundtrip(&mut sess.state)?;+ Ok(())+}++/// Press-drag-release on a native Wayland toplevel. Emits one button press at+/// `(from_x, from_y)`, then `steps` interpolated motion events along the+/// straight segment to `(to_x, to_y)`, then a release. Coordinates are+/// output-relative; window-local coords need the EIS inject socket+/// (`CUA_INJECT_SOCKET`).+pub fn drag(+ window_id: u64,+ from_x: i32,+ from_y: i32,+ to_x: i32,+ to_y: i32,+ steps: u32,+ button: u8,+) -> anyhow::Result<()> {+ let mut sess = open_vptr_session(Some(window_id as u32))?;+ let (w, h) = (sess.output_w, sess.output_h);+ let btn = evdev_pointer_button(button);+ let clamp_xy = |x: i32, y: i32| -> (u32, u32) {+ (+ x.clamp(0, w as i32 - 1) as u32,+ y.clamp(0, h as i32 - 1) as u32,+ )+ };+ let (fx, fy) = clamp_xy(from_x, from_y);+ sess.vptr.motion_absolute(0, fx, fy, w, h);+ sess.vptr.frame();+ sess.vptr.button(0, btn, ButtonState::Pressed);+ sess.vptr.frame();+ sess.queue.roundtrip(&mut sess.state)?;+ let n = steps.max(1);+ for s in 1..=n {+ let t = s as f64 / n as f64;+ let ix = (from_x as f64 + (to_x - from_x) as f64 * t).round() as i32;+ let iy = (from_y as f64 + (to_y - from_y) as f64 * t).round() as i32;+ let (cx, cy) = clamp_xy(ix, iy);+ sess.vptr.motion_absolute(0, cx, cy, w, h);+ sess.vptr.frame();+ sess.queue.roundtrip(&mut sess.state)?;+ std::thread::sleep(std::time::Duration::from_millis(8));+ }+ let (tx, ty) = clamp_xy(to_x, to_y);+ sess.vptr.motion_absolute(0, tx, ty, w, h);+ sess.vptr.frame();+ sess.vptr.button(0, btn, ButtonState::Released);+ sess.vptr.frame();+ // Sync the synthetic-cursor registry with the drag endpoint so a+ // subsequent `get_cursor_position` reports where we left the pointer.+ record_synth_cursor(tx as i32, ty as i32);+ sess.queue.roundtrip(&mut sess.state)?;+ sess.vptr.destroy();+ sess.queue.roundtrip(&mut sess.state)?;Ok(())}@@ -424,6 +1012,59 @@ pub fn press_key(key: &str) -> anyhow::Result<()> {Ok(())}+/// Press a key combination (modifiers + final key) via `wtype`. Each modifier+/// is pressed before the key, then released after, exactly like+/// `wtype -M ctrl -M shift -k key -m shift -m ctrl`. Unknown values pass+/// straight to wtype's `-k` so single-character keys and X keysym names work+/// as-is. This is the Wayland equivalent of the X11 `send_key` modifier mask.+pub fn hotkey(keys: &[String]) -> anyhow::Result<()> {+ let (mods, final_key) = partition_modifiers(keys)?;+ let keysym = key_to_keysym(&final_key);+ let mut args: Vec<String> = Vec::new();+ for m in &mods {+ args.push("-M".into());+ args.push(m.clone());+ }+ args.push("-k".into());+ args.push(keysym.clone());+ // Release modifiers in reverse press order.+ for m in mods.iter().rev() {+ args.push("-m".into());+ args.push(m.clone());+ }+ let out = std::process::Command::new("wtype").args(&args).output()?;+ if !out.status.success() {+ anyhow::bail!(+ "wtype {} failed: {}",+ args.join(" "),+ String::from_utf8_lossy(&out.stderr)+ );+ }+ Ok(())+}++/// Split a `keys` array into wtype-compatible modifier names and a single+/// final key. Recognised modifier inputs: ctrl/control, alt, shift,+/// super/meta/cmd/command/win/windows. The final key must be the one+/// non-modifier in the list.+fn partition_modifiers(keys: &[String]) -> anyhow::Result<(Vec<String>, String)> {+ let mut mods: Vec<String> = Vec::new();+ let mut non_mods: Vec<String> = Vec::new();+ for k in keys {+ match k.to_ascii_lowercase().as_str() {+ "ctrl" | "control" => mods.push("ctrl".into()),+ "alt" => mods.push("alt".into()),+ "shift" => mods.push("shift".into()),+ "super" | "meta" | "cmd" | "command" | "win" | "windows" => mods.push("logo".into()),+ _ => non_mods.push(k.clone()),+ }+ }+ let final_key = non_mods.last().cloned().ok_or_else(|| {+ anyhow::anyhow!("hotkey requires at least one non-modifier key")+ })?;+ Ok((mods, final_key))+}+/// Map cua key names to X keysym names that `wtype -k` understands. Unknown/// values pass through (single characters and valid keysym names work as-is).fn key_to_keysym(key: &str) -> String {@@ -510,6 +1151,20 @@ pub fn app_id_for_window(window_id: u64) -> Option<String> {.filter(|s| !s.is_empty())}+/// Resolve the WM_CLASS-equivalent (instance, class) pair for a window. On+/// X11 reads `WM_CLASS`; on Wayland reuses [`app_id_for_window`] and returns+/// `(app_id, app_id)` — the closest analogue, since foreign-toplevel exposes+/// a single app_id and not the X11 instance/class split. Used by terminal+/// emulator detection on `is_terminal_window` so Ghostty / kitty / alacritty+/// are recognised on Wayland too.+pub fn wm_class_dispatch(window_id: u64) -> Option<(String, String)> {+ if is_wayland() {+ let app = app_id_for_window(window_id)?;+ return Some((app.clone(), app));+ }+ crate::x11::wm_class_for_window(window_id)+}+fn to_hex(s: &str) -> String {s.bytes().map(|b| format!("{b:02x}")).collect()}@@ -619,3 +1274,97 @@ pub fn list_windows_dispatch(filter_pid: Option<u32>) -> Vec<WindowInfo> {}crate::x11::list_windows(filter_pid)}++/// Snapshot of which wlroots manager globals the running compositor advertises.+/// Used by the `health_report` Wayland backend probe to distinguish a working+/// session from one missing screencopy or virtual-pointer support.+#[derive(Default, Clone, Debug)]+pub struct WaylandManagers {+ pub foreign_toplevel: bool,+ pub screencopy: bool,+ pub virtual_pointer: bool,+ pub wl_shm: bool,+ /// Staging `ext-image-copy-capture-v1` manager — sway 1.10+, labwc+ /// 0.8+, niri, KDE 6.2+, GNOME mutter 47+.+ pub ext_image_copy_capture: bool,+ /// Companion `ext-output-image-capture-source-v1` source manager —+ /// required to capture a `wl_output` via the staging protocol.+ pub ext_output_image_capture_source: bool,+}++/// Perform a single registry roundtrip and report which of the manager+/// interfaces the doctor cares about advertise themselves. Returns `Err` only+/// when we can't even open a Wayland connection — a successful connect with+/// no managers still resolves to an all-false snapshot.+pub fn probe_managers() -> anyhow::Result<WaylandManagers> {+ let conn = Connection::connect_to_env()?;+ let mut queue = conn.new_event_queue::<State>();+ let qh = queue.handle();+ conn.display().get_registry(&qh, ());+ let mut state = State::default();+ queue.roundtrip(&mut state)?;+ // Reuse the existing State for wlroots managers, then do a parallel+ // probe for staging ext-image-copy-capture interfaces by walking the+ // raw registry events (no binding required — we only need presence).+ let ext_probe = probe_ext_interfaces().unwrap_or_default();+ Ok(WaylandManagers {+ foreign_toplevel: state.manager.is_some(),+ screencopy: state.scrcopy_manager.is_some(),+ virtual_pointer: state.vptr_manager.is_some(),+ wl_shm: state.shm.is_some(),+ ext_image_copy_capture: ext_probe.image_copy_capture,+ ext_output_image_capture_source: ext_probe.output_image_capture_source,+ })+}++#[derive(Default, Clone, Copy, Debug)]+struct ExtInterfaceProbe {+ image_copy_capture: bool,+ output_image_capture_source: bool,+}++/// Probe registry for `ext-image-copy-capture-v1` + companion source manager+/// presence without binding them. Cheap (one roundtrip) and side-effect+/// free.+fn probe_ext_interfaces() -> anyhow::Result<ExtInterfaceProbe> {+ let conn = Connection::connect_to_env()?;+ let mut queue = conn.new_event_queue::<ExtProbeState>();+ let qh = queue.handle();+ conn.display().get_registry(&qh, ());+ let mut state = ExtProbeState::default();+ queue.roundtrip(&mut state)?;+ Ok(state.probe)+}++#[derive(Default)]+struct ExtProbeState {+ probe: ExtInterfaceProbe,+}++impl Dispatch<wl_registry::WlRegistry, ()> for ExtProbeState {+ fn event(+ state: &mut Self,+ _: &wl_registry::WlRegistry,+ event: wl_registry::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ if let wl_registry::Event::Global { interface, .. } = event {+ match interface.as_str() {+ "ext_image_copy_capture_manager_v1" => {+ state.probe.image_copy_capture = true;+ }+ "ext_output_image_capture_source_manager_v1" => {+ state.probe.output_image_capture_source = true;+ }+ _ => {}+ }+ }+ }+}++// Suppress dead-code warning for the unused BTN_LEFT alias kept for backward+// compatibility with earlier slice constants.+#[allow(dead_code)]+const _BTN_LEFT_ALIAS: u32 = BTN_LEFT;diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rsnew file mode 100644index 0000000000..7ffac0a279--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs@@ -0,0 +1,609 @@+//! Native-Wayland agent-cursor overlay via `zwlr_layer_shell_v1`.+//!+//! Replaces the X11-only `overlay.rs` render loop on wlroots compositors+//! (sway, labwc, kwin 5.27+, hyprland) by creating a full-screen,+//! click-through, always-on-top `wl_surface` anchored to the first output+//! via `zwlr_layer_shell_v1`. The surface renders the same gradient-arrow+//! cursor as the X11 path by sharing `cursor_overlay::RenderStateCore` —+//! bloom, click-pulse, idle-fade, and motion all work identically.+//!+//! GNOME mutter does not expose `zwlr_layer_shell_v1` — those sessions+//! either fall through to the X11 path (XWayland) or the nested-compositor+//! mode that spawns labwc internally.+//!+//! Architecture mirrors the existing `wayland/persistent_vptr.rs`: one+//! owner thread (`cua-overlay-wl`) holds the wayland Connection ++//! EventQueue + layer surface; commands flow in over a `crossbeam-channel`.+//! The render core is ticked at ~60Hz via a calloop timer so motion ++//! spring physics + click pulse advance smoothly even when no new+//! Position command has arrived.++use std::collections::HashMap;+use std::sync::OnceLock;+use std::thread;+use std::time::Instant;++use crossbeam_channel::{bounded, Receiver, Sender};+use cursor_overlay::{CursorConfig, OverlayCommand, OverlayMsg, CursorKey, RenderStateCore};+use wayland_client::{+ protocol::{+ wl_buffer::WlBuffer,+ wl_compositor::WlCompositor,+ wl_output::WlOutput,+ wl_region::WlRegion,+ wl_registry,+ wl_shm::{self, WlShm},+ wl_shm_pool::WlShmPool,+ wl_surface::WlSurface,+ },+ Connection, Dispatch, Proxy, QueueHandle,+};+use wayland_protocols_wlr::layer_shell::v1::client::{+ zwlr_layer_shell_v1::{self, Layer, ZwlrLayerShellV1},+ zwlr_layer_surface_v1::{self, Anchor, KeyboardInteractivity, ZwlrLayerSurfaceV1},+};++/// Commands the overlay owner thread accepts. The richer commands the+/// cross-platform [`RenderStateCore`] understands (MoveTo, ClickPulse,+/// SetPressed) are forwarded as-is so the layer-shell overlay matches the+/// X11 visual: bloom + animated arrow + click pulse + press ring.+enum WlOverlayCmd {+ Cmd { key: CursorKey, cmd: OverlayCommand },+ Remove { key: CursorKey },+ Shutdown,+}++static TX: OnceLock<Sender<WlOverlayCmd>> = OnceLock::new();++fn tx() -> Option<&'static Sender<WlOverlayCmd>> {+ TX.get()+}++/// Lazily start the owner thread. Idempotent — safe to call from every+/// MCP tool invocation; subsequent calls are no-ops.+pub fn ensure_started() {+ TX.get_or_init(|| {+ let (tx, rx) = bounded::<WlOverlayCmd>(64);+ thread::Builder::new()+ .name("cua-overlay-wl".into())+ .spawn(move || {+ if let Err(e) = owner_thread(rx) {+ tracing::warn!("cua-overlay-wl thread exited with error: {e}");+ }+ })+ .expect("spawn cua-overlay-wl thread");+ tx+ });+}++/// Translate a generic [`OverlayMsg`] (the cross-platform command shape)+/// to the layer-shell owner thread. The owner-thread render core consumes+/// every variant the X11 path handles; only `ShowFocusRect` (macOS-only)+/// is silently dropped here.+pub fn forward(msg: &OverlayMsg) -> bool {+ // Lazy startup: spawning the layer-shell owner thread + connecting to+ // the Wayland compositor takes 100-300ms. Doing that at cua-driver mcp+ // boot (the old eager-init path) was tipping the borderline CI+ // cursor-click-gif test over its 20s budget. ensure_started is+ // idempotent so calling it on every forward is fine — the OnceLock+ // bypasses the spawn after the first call.+ ensure_started();+ let Some(tx) = tx() else { return false };+ match msg {+ OverlayMsg::Remove(k) => {+ let _ = tx.try_send(WlOverlayCmd::Remove { key: k.clone() });+ true+ }+ OverlayMsg::Cmd(kc) => {+ if matches!(&kc.cmd, OverlayCommand::ShowFocusRect(_)) {+ return false;+ }+ let _ = tx.try_send(WlOverlayCmd::Cmd {+ key: kc.key.clone(),+ cmd: kc.cmd.clone(),+ });+ true+ }+ }+}++/// Cleanly stop the owner thread. Tests use this; production code typically+/// lets the thread die at process exit.+pub fn shutdown() {+ if let Some(tx) = tx() {+ let _ = tx.send(WlOverlayCmd::Shutdown);+ }+}++// ── owner thread ─────────────────────────────────────────────────────────++struct OverlayState {+ compositor: Option<WlCompositor>,+ shm: Option<WlShm>,+ layer_shell: Option<ZwlrLayerShellV1>,+ output: Option<WlOutput>,+ output_w: u32,+ output_h: u32,+ surface: Option<WlSurface>,+ layer_surface: Option<ZwlrLayerSurfaceV1>,+ configured: bool,+ /// Cross-platform render core: position, animation, gradient arrow,+ /// bloom, click pulse, idle-fade. Shared verbatim with the X11 path.+ core: RenderStateCore,+ /// In-flight wl_shm buffers awaiting `wl_buffer.release` from the+ /// compositor. Keyed by `WlBuffer` object id; value is the+ /// `(mmap ptr, mmap size, memfd fd)` triple that must be unmapped ++ /// closed once the compositor signals it's done with the buffer.+ /// Replaces the per-redraw `mem::forget` leak: the previous frame's+ /// memory is reclaimed as soon as the compositor releases it.+ pending_buffers: HashMap<u32, (*mut libc::c_void, usize, i32)>,+}++// SAFETY: the raw pointers in pending_buffers point at mmap regions owned+// exclusively by this thread (the owner thread). OverlayState is never+// shared across threads — wayland-client's EventQueue<State> is !Send so+// it stays pinned to the owner thread. The Send/Sync bounds wayland-client+// requires for State types apply to the struct as a whole, hence the+// explicit assertion.+unsafe impl Send for OverlayState {}++impl Default for OverlayState {+ fn default() -> Self {+ Self {+ compositor: None,+ shm: None,+ layer_shell: None,+ output: None,+ output_w: 0,+ output_h: 0,+ surface: None,+ layer_surface: None,+ configured: false,+ core: RenderStateCore::new(CursorConfig::default()),+ pending_buffers: HashMap::new(),+ }+ }+}++fn dbg(msg: &str) {+ if std::env::var_os("CUA_OVERLAY_DEBUG").is_some() {+ eprintln!("[cua-overlay-wl] {msg}");+ }+}++fn owner_thread(rx: Receiver<WlOverlayCmd>) -> anyhow::Result<()> {+ let conn = Connection::connect_to_env()?;+ let mut queue = conn.new_event_queue::<OverlayState>();+ let qh = queue.handle();+ let _registry = conn.display().get_registry(&qh, ());++ let mut state = OverlayState::default();+ queue.roundtrip(&mut state)?;+ for _ in 0..3 {+ queue.roundtrip(&mut state)?;+ }++ let compositor = state+ .compositor+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_compositor"))?;+ let shm = state+ .shm+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_shm"))?;+ let layer_shell = state+ .layer_shell+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose zwlr_layer_shell_v1"))?;+ let output = state+ .output+ .clone()+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_output"))?;++ // Build the layer surface: fullscreen, overlay layer, click-through.+ let surface = compositor.create_surface(&qh, ());+ let layer_surface = layer_shell.get_layer_surface(+ &surface,+ Some(&output),+ Layer::Overlay,+ "cua-agent-cursor".to_string(),+ &qh,+ (),+ );+ // Anchor to all four edges = full screen.+ layer_surface.set_anchor(Anchor::Top | Anchor::Bottom | Anchor::Left | Anchor::Right);+ layer_surface.set_size(0, 0);+ layer_surface.set_exclusive_zone(-1);+ layer_surface.set_keyboard_interactivity(KeyboardInteractivity::None);++ // Click-through: empty input region.+ let region: WlRegion = compositor.create_region(&qh, ());+ surface.set_input_region(Some(®ion));+ region.destroy();++ state.surface = Some(surface);+ state.layer_surface = Some(layer_surface);++ // First commit kicks off the configure handshake.+ if let Some(s) = state.surface.as_ref() {+ s.commit();+ }++ // Wait for the first configure event so we know the output dimensions+ // before drawing.+ for _ in 0..10 {+ queue.roundtrip(&mut state)?;+ if state.configured && state.output_w > 0 && state.output_h > 0 {+ break;+ }+ std::thread::sleep(std::time::Duration::from_millis(50));+ }++ if !state.configured {+ anyhow::bail!("layer surface never received configure event");+ }+ dbg(&format!("configured: w={} h={}", state.output_w, state.output_h));++ // Main loop. Tick the render core at ~60Hz so motion + spring physics+ // + click pulse animate smoothly; redraw every tick when the cursor is+ // visible. Commands arriving via the channel update the render core+ // first, then the next tick paints the result.+ redraw(&mut state, &shm, &qh)?;+ queue.roundtrip(&mut state)?;++ let frame_dur = std::time::Duration::from_millis(16);+ let mut last_tick = Instant::now();+ loop {+ // Drain all pending commands without blocking.+ let mut shutdown = false;+ loop {+ match rx.try_recv() {+ Ok(WlOverlayCmd::Shutdown) => { shutdown = true; break; }+ Ok(WlOverlayCmd::Cmd { cmd, .. }) => {+ // Seed: if the cursor is still at the off-screen sentinel+ // `(-200, -200)` from `RenderStateCore::new`, snap to a+ // point near the MoveTo / SnapTo target so the spring+ // animation starts on-screen. Mirrors X11 overlay.rs's+ // `seed_start_if_sentinel` helper — without it, the+ // spring oscillates around the sentinel and the cursor+ // never reaches the screen.+ let seed_target = match &cmd {+ OverlayCommand::MoveTo { x, y, .. }+ | OverlayCommand::SnapTo { x, y, .. }+ | OverlayCommand::ClickPulse { x, y } => Some((*x, *y)),+ _ => None,+ };+ if let Some((tx, ty)) = seed_target {+ if state.core.pos.0 < -50.0 {+ const SEED_OFFSET: f64 = 16.0;+ let sx = (tx - SEED_OFFSET).max(2.0);+ let sy = (ty - SEED_OFFSET).max(2.0);+ state.core.pos = (sx, sy);+ }+ }+ // apply_command_base consumes every variant the X11+ // path handles. `move_to_snap_sentinel` / `click_pulse+ // _sentinel_only` are both `false` here — same as X11.+ let _ = state.core.apply_command_base(cmd, false, false);+ }+ Ok(WlOverlayCmd::Remove { .. }) => {+ // Single-cursor overlay: removing the active cursor+ // hides it. Multi-cursor wlroots support can layer on+ // top of this in a follow-up if needed.+ state.core.visible = false;+ }+ Err(crossbeam_channel::TryRecvError::Empty) => break,+ Err(crossbeam_channel::TryRecvError::Disconnected) => { shutdown = true; break; }+ }+ }+ if shutdown { break; }++ // Tick animation forward and repaint if anything changed.+ let now = Instant::now();+ let dt = now.duration_since(last_tick).as_secs_f64().min(0.05);+ last_tick = now;+ state.core.tick_motion(dt);+ if state.configured {+ redraw(&mut state, &shm, &qh)?;+ }+ queue.dispatch_pending(&mut state)?;++ // Sleep for the remainder of the frame budget so the loop doesn't+ // spin. Channel-driven wakeups would be lower-latency, but layer+ // overlays only need to keep up with display refresh.+ let elapsed = last_tick.elapsed();+ if elapsed < frame_dur {+ std::thread::sleep(frame_dur - elapsed);+ }+ }++ if let Some(ls) = state.layer_surface.take() {+ ls.destroy();+ }+ if let Some(s) = state.surface.take() {+ s.destroy();+ }+ queue.roundtrip(&mut state)?;+ Ok(())+}++/// Render one cursor frame into a fresh wl_shm ARGB8888 buffer and attach+/// it to the layer surface.+///+/// Pipeline:+/// 1. Allocate a memfd-backed wl_shm pool sized at output_w × output_h.+/// 2. Paint the cross-platform cursor (bloom + click pulse + gradient+/// arrow) into a `tiny_skia::Pixmap` via `cursor_overlay::paint_cursor`+/// — same call the X11 path uses.+/// 3. Channel-swap RGBA → BGRA into the wl_shm buffer (wl_shm Argb8888+/// is little-endian BGRA in memory). This is the inverse of the swap+/// in `ext_screencopy::encode_buffer_to_png`.+/// 4. Attach + damage + commit on the layer surface.+///+/// When the cursor is hidden (`core.visible == false`, idle-faded, or+/// off-screen sentinel) the pixmap is all zeros — the surface remains+/// transparent and click-through.+fn redraw(state: &mut OverlayState, shm: &WlShm, qh: &QueueHandle<OverlayState>) -> anyhow::Result<()> {+ let Some(surface) = state.surface.as_ref() else { return Ok(()) };+ let w = state.output_w.max(1);+ let h = state.output_h.max(1);+ let stride = w as i32 * 4;+ let size = (stride as usize) * (h as usize);++ // Reuses the same anon_shm pattern as the screencopy path in mod.rs.+ let (fd, ptr) = super::anon_shm(size)+ .map_err(|e| anyhow::anyhow!("overlay shm allocation failed: {e}"))?;++ // SAFETY: ptr came from mmap of `size` bytes, lifetime bounded to this+ // function.+ let pixels: &mut [u8] = unsafe { std::slice::from_raw_parts_mut(ptr as *mut u8, size) };++ // Paint the cursor into a tiny_skia pixmap. paint_cursor early-returns+ // when the cursor is hidden / off-screen / idle-faded, so the pixmap+ // is left fully transparent in those cases (which is also what we want+ // for the click-through layer surface).+ let pm_result = tiny_skia::Pixmap::new(w, h);+ let mut pm = match pm_result {+ Some(p) => p,+ // tiny_skia::Pixmap::new only fails on OOM at sizes that fit u32.+ // We refuse to fall back to a 1x1 pixmap because the subsequent+ // RGBA → BGRA loop indexes `src[i+3]` over the full `size` range —+ // a 1x1 source would crash. Surface the allocation failure+ // properly instead.+ None => anyhow::bail!(+ "tiny_skia::Pixmap::new({w}, {h}) failed — out of memory for the overlay buffer"+ ),+ };+ // backing_scale=1.0 matches the X11 path; per-output Wayland scale is+ // a follow-up (would consume `wl_output.scale` and `preferred_buffer+ // _scale` from wl_surface v6).+ cursor_overlay::paint_cursor(&mut pm, &state.core, 0.0, 0.0, None, 1.0);++ // CUA_OVERLAY_DEBUG=1 paints a 60x60 magenta square at the cursor's+ // current pos on top of the gradient arrow. Useful when validating+ // layer-shell visibility on a new compositor — the gradient arrow is+ // small at native scale and easy to miss in a screenshot, while the+ // magenta block is impossible to miss.+ if std::env::var_os("CUA_OVERLAY_DEBUG").is_some() {+ let (cx, cy) = state.core.pos;+ let cx = cx as i32;+ let cy = cy as i32;+ let half = 30i32;+ for dy in -half..half {+ for dx in -half..half {+ let px = cx + dx;+ let py = cy + dy;+ if px < 0 || py < 0 || px >= w as i32 || py >= h as i32 {+ continue;+ }+ let off = ((py as usize) * (w as usize) + (px as usize)) * 4;+ pm.data_mut()[off] = 0xFF; // R+ pm.data_mut()[off + 1] = 0x00; // G+ pm.data_mut()[off + 2] = 0xFF; // B+ pm.data_mut()[off + 3] = 0xFF; // A+ }+ }+ }++// RGBA → BGRA channel swap. tiny_skia stores pixels as RGBA8888+ // (premultiplied); wl_shm Argb8888 is little-endian = BGRA in memory.+ // Mirrors the inverse swap in ext_screencopy::encode_buffer_to_png.+ let src = pm.data();+ for i in (0..size).step_by(4) {+ // pm.data() is already RGBA premultiplied; just swap R↔B.+ pixels[i] = src[i + 2]; // B ← R+ pixels[i + 1] = src[i + 1]; // G+ pixels[i + 2] = src[i]; // R ← B+ pixels[i + 3] = src[i + 3]; // A+ }++ use std::os::fd::AsFd as _;+ let pool_fd = unsafe { super::borrowed_fd(fd) };+ let pool: WlShmPool = shm.create_pool(pool_fd.as_fd(), size as i32, qh, ());+ let buffer: WlBuffer = pool.create_buffer(+ 0,+ w as i32,+ h as i32,+ stride,+ wl_shm::Format::Argb8888,+ qh,+ (),+ );++ // Track the (mmap, fd) by buffer object id so the wl_buffer.release+ // event Dispatch handler can clean up exactly when the compositor is+ // done with the underlying memory — no leak, no use-after-free.+ let buffer_id = buffer.id().protocol_id();+ state.pending_buffers.insert(buffer_id, (ptr, size, fd));++ dbg(&format!("redraw w={w} h={h} stride={stride} buf_id={buffer_id} pos=({:.1},{:.1}) visible={}", state.core.pos.0, state.core.pos.1, state.core.visible));+ surface.attach(Some(&buffer), 0, 0);+ surface.damage_buffer(0, 0, w as i32, h as i32);+ surface.commit();+ pool.destroy();+ Ok(())+}++// ── Wayland Dispatch impls ───────────────────────────────────────────────++impl Dispatch<wl_registry::WlRegistry, ()> for OverlayState {+ fn event(+ state: &mut Self,+ registry: &wl_registry::WlRegistry,+ event: wl_registry::Event,+ _data: &(),+ _conn: &Connection,+ qh: &QueueHandle<Self>,+ ) {+ if let wl_registry::Event::Global { name, interface, version } = event {+ match interface.as_str() {+ "wl_compositor" => {+ state.compositor = Some(registry.bind::<WlCompositor, _, _>(+ name, version.min(6), qh, ()));+ }+ "wl_shm" => {+ state.shm = Some(registry.bind::<WlShm, _, _>(name, version.min(1), qh, ()));+ }+ "wl_output" => {+ if state.output.is_none() {+ state.output = Some(registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ()));+ }+ }+ "zwlr_layer_shell_v1" => {+ state.layer_shell = Some(registry.bind::<ZwlrLayerShellV1, _, _>(+ name, version.min(4), qh, ()));+ }+ _ => {}+ }+ }+ }+}++impl Dispatch<WlCompositor, ()> for OverlayState {+ fn event(+ _state: &mut Self,+ _: &WlCompositor,+ _: <WlCompositor as wayland_client::Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {}+}++impl Dispatch<WlShm, ()> for OverlayState {+ fn event(+ _state: &mut Self,+ _: &WlShm,+ _: <WlShm as wayland_client::Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {}+}++impl Dispatch<WlOutput, ()> for OverlayState {+ fn event(+ state: &mut Self,+ _: &WlOutput,+ event: <WlOutput as wayland_client::Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ use wayland_client::protocol::wl_output;+ if let wl_output::Event::Mode { width, height, .. } = event {+ if width > 0 && height > 0 {+ state.output_w = width as u32;+ state.output_h = height as u32;+ }+ }+ }+}++impl Dispatch<ZwlrLayerShellV1, ()> for OverlayState {+ fn event(+ _state: &mut Self,+ _: &ZwlrLayerShellV1,+ _: <ZwlrLayerShellV1 as wayland_client::Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {}+}++impl Dispatch<ZwlrLayerSurfaceV1, ()> for OverlayState {+ fn event(+ state: &mut Self,+ layer: &ZwlrLayerSurfaceV1,+ event: <ZwlrLayerSurfaceV1 as wayland_client::Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ if let zwlr_layer_surface_v1::Event::Configure { serial, width, height } = event {+ layer.ack_configure(serial);+ if width > 0 {+ state.output_w = width;+ }+ if height > 0 {+ state.output_h = height;+ }+ state.configured = true;+ }+ }+}++impl Dispatch<WlSurface, ()> for OverlayState {+ fn event(+ _: &mut Self,+ _: &WlSurface,+ _: <WlSurface as wayland_client::Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {}+}++impl Dispatch<WlShmPool, ()> for OverlayState {+ fn event(+ _: &mut Self,+ _: &WlShmPool,+ _: <WlShmPool as wayland_client::Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {}+}++impl Dispatch<WlBuffer, ()> for OverlayState {+ fn event(+ state: &mut Self,+ buffer: &WlBuffer,+ event: <WlBuffer as wayland_client::Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {+ use wayland_client::protocol::wl_buffer;+ if matches!(event, wl_buffer::Event::Release) {+ // Compositor is done with the underlying mmap. Free it ++ // close the memfd + destroy the wayland object.+ let id = buffer.id().protocol_id();+ if let Some((ptr, size, fd)) = state.pending_buffers.remove(&id) {+ super::cleanup_mmap(ptr, size, fd);+ }+ buffer.destroy();+ }+ }+}++impl Dispatch<WlRegion, ()> for OverlayState {+ fn event(+ _: &mut Self,+ _: &WlRegion,+ _: <WlRegion as wayland_client::Proxy>::Event,+ _: &(),+ _: &Connection,+ _: &QueueHandle<Self>,+ ) {}+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rsnew file mode 100644index 0000000000..7f3eaa2bcd--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs@@ -0,0 +1,287 @@+//! Persistent virtual-pointer for stateful `mouse_button_down` / `mouse_drag` /+//! `mouse_button_up`.+//!+//! The non-persistent path in [`crate::wayland::click`] opens its own+//! `ZwlrVirtualPointerV1`, presses, releases, drops the connection — useful+//! for one-shot clicks but useless for held-button drags: each tool call+//! emits a fresh device whose press/release pair is matched by the+//! compositor, so apps that distinguish a real drag (press, motion+, release)+//! from a series of clicks (press, release, press, release, …) miss the+//! drag entirely.+//!+//! This module keeps the virtual-pointer alive across tool calls. A single+//! owner thread per process owns one Wayland `Connection`, one `EventQueue`,+//! and a map of `cursor_id -> ActivePointer`. Commands are sent over a+//! `crossbeam-channel`; replies come back on a per-call reply channel so the+//! caller blocks until the compositor has roundtripped.+//!+//! Lifecycle:+//! - First `press` for a cursor_id binds a fresh `ZwlrVirtualPointerV1`,+//! activates the foreign-toplevel target window once, presses the button,+//! adds to the held-button set, roundtrips.+//! - Subsequent `move_to` calls emit `motion_absolute` on the same vptr (no+//! activate — would steal focus mid-drag) and roundtrip.+//! - `release` emits a button release, removes from the held set; if the+//! set is empty the vptr is destroyed and the map entry dropped.+//! - On `Connection` roundtrip failure (compositor restart / disconnect)+//! the owner thread tears down its connection and accepts the next+//! command on a fresh one, emitting a typed error for the in-flight call.++use std::collections::{HashMap, HashSet};+use std::sync::OnceLock;+use std::thread;++use crossbeam_channel::{bounded, Receiver, Sender};+use wayland_client::{+ protocol::wl_pointer::ButtonState,+ Connection, Proxy,+};+use wayland_protocols_wlr::virtual_pointer::v1::client::zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1;++use super::{open_vptr_session, evdev_pointer_button};++/// One in-flight command from the public API to the owner thread.+enum Cmd {+ Press {+ cursor_id: String,+ window_id: u64,+ x: i32,+ y: i32,+ button: u8,+ reply: Sender<anyhow::Result<()>>,+ },+ MoveTo {+ cursor_id: String,+ x: i32,+ y: i32,+ reply: Sender<anyhow::Result<()>>,+ },+ Release {+ cursor_id: String,+ button: u8,+ reply: Sender<anyhow::Result<()>>,+ },+ /// Drop the entry for a cursor_id without sending wire events — used to+ /// recover when the compositor disconnected mid-life.+ Forget {+ cursor_id: String,+ reply: Sender<anyhow::Result<()>>,+ },+}++/// State held inside the owner thread for one cursor_id.+struct ActivePointer {+ vptr: ZwlrVirtualPointerV1,+ /// evdev codes of buttons currently held down. When this set becomes+ /// empty the vptr is destroyed and the entry dropped from the map.+ held: HashSet<u32>,+ /// Output extent at session open time — needed for motion_absolute.+ out_w: u32,+ out_h: u32,+}++/// Process-global command channel into the owner thread. Lazily started on+/// first use.+static TX: OnceLock<Sender<Cmd>> = OnceLock::new();++fn tx() -> &'static Sender<Cmd> {+ TX.get_or_init(|| {+ let (tx, rx) = bounded::<Cmd>(32);+ thread::Builder::new()+ .name("cua-persistent-vptr".into())+ .spawn(move || owner_thread(rx))+ .expect("spawn cua-persistent-vptr thread");+ tx+ })+}++fn owner_thread(rx: Receiver<Cmd>) {+ let mut active: HashMap<String, ActivePointer> = HashMap::new();+ while let Ok(cmd) = rx.recv() {+ match cmd {+ Cmd::Press { cursor_id, window_id, x, y, button, reply } => {+ let r = handle_press(&mut active, &cursor_id, window_id, x, y, button);+ let _ = reply.send(r);+ }+ Cmd::MoveTo { cursor_id, x, y, reply } => {+ let r = handle_move(&mut active, &cursor_id, x, y);+ let _ = reply.send(r);+ }+ Cmd::Release { cursor_id, button, reply } => {+ let r = handle_release(&mut active, &cursor_id, button);+ let _ = reply.send(r);+ }+ Cmd::Forget { cursor_id, reply } => {+ if let Some(p) = active.remove(&cursor_id) {+ p.vptr.destroy();+ }+ let _ = reply.send(Ok(()));+ }+ }+ }+}++fn handle_press(+ active: &mut HashMap<String, ActivePointer>,+ cursor_id: &str,+ window_id: u64,+ x: i32,+ y: i32,+ button: u8,+) -> anyhow::Result<()> {+ // Open a fresh session for this press — this binds the seat, the foreign-+ // toplevel manager, activates the target window, and creates a new vptr.+ // Keep the (out_w, out_h) but drop the queue + state at end of scope; the+ // vptr itself remains alive (Wayland objects survive their original queue+ // as long as the Connection is alive).+ let mut sess = open_vptr_session(Some(window_id as u32))?;+ let (w, h) = (sess.output_w, sess.output_h);+ let px = x.clamp(0, w as i32 - 1) as u32;+ let py = y.clamp(0, h as i32 - 1) as u32;+ let btn = evdev_pointer_button(button);++ sess.vptr.motion_absolute(0, px, py, w, h);+ sess.vptr.frame();+ sess.vptr.button(0, btn, ButtonState::Pressed);+ sess.vptr.frame();+ sess.queue.roundtrip(&mut sess.state)?;++ // Take ownership of the vptr handle by extracting it from the session.+ // ZwlrVirtualPointerV1 is a Wayland proxy — cloning it gives another+ // handle to the same wire object; destroying it sends the destructor.+ let vptr = sess.vptr.clone();+ // Persist the live connection so the proxy stays valid after this fn returns+ // (the session goes out of scope; we need the conn alive).+ // We do this by leaking the connection into a process-static slot keyed by+ // cursor_id. Subsequent commands on the same cursor reuse this conn.+ persist_conn(cursor_id, sess.conn);++ let mut held = HashSet::new();+ held.insert(btn);+ active.insert(cursor_id.to_string(), ActivePointer { vptr, held, out_w: w, out_h: h });+ Ok(())+}++fn handle_move(+ active: &mut HashMap<String, ActivePointer>,+ cursor_id: &str,+ x: i32,+ y: i32,+) -> anyhow::Result<()> {+ let entry = active.get_mut(cursor_id).ok_or_else(|| {+ anyhow::anyhow!("no held mouse button for cursor '{cursor_id}'; call mouse_button_down first")+ })?;+ let px = x.clamp(0, entry.out_w as i32 - 1) as u32;+ let py = y.clamp(0, entry.out_h as i32 - 1) as u32;+ entry.vptr.motion_absolute(0, px, py, entry.out_w, entry.out_h);+ entry.vptr.frame();+ roundtrip_on_persistent(cursor_id)?;+ Ok(())+}++fn handle_release(+ active: &mut HashMap<String, ActivePointer>,+ cursor_id: &str,+ button: u8,+) -> anyhow::Result<()> {+ let btn = evdev_pointer_button(button);+ let drop_entry = {+ let entry = active.get_mut(cursor_id).ok_or_else(|| {+ anyhow::anyhow!("no held mouse button for cursor '{cursor_id}'")+ })?;+ entry.vptr.button(0, btn, ButtonState::Released);+ entry.vptr.frame();+ roundtrip_on_persistent(cursor_id)?;+ entry.held.remove(&btn);+ entry.held.is_empty()+ };+ if drop_entry {+ if let Some(p) = active.remove(cursor_id) {+ p.vptr.destroy();+ roundtrip_on_persistent(cursor_id).ok();+ }+ forget_conn(cursor_id);+ }+ Ok(())+}++// Process-static slots for Connection + EventQueue keyed by cursor_id. The+// EventQueue is !Send but we only touch these on the owner thread, so wrap+// in a thread-local-by-construction pattern: store inside the same map so+// the owner thread is the sole accessor.+//+// We use a per-thread static rather than a Mutex<HashMap> because the owner+// thread is the only accessor (no contention possible).+thread_local! {+ static CONNS: std::cell::RefCell<HashMap<String, (Connection, wayland_client::EventQueue<super::State>)>>+ = std::cell::RefCell::new(HashMap::new());+}++fn persist_conn(cursor_id: &str, conn: Connection) {+ let queue = conn.new_event_queue::<super::State>();+ CONNS.with(|c| { c.borrow_mut().insert(cursor_id.to_string(), (conn, queue)); });+}++fn forget_conn(cursor_id: &str) {+ CONNS.with(|c| { c.borrow_mut().remove(cursor_id); });+}++fn roundtrip_on_persistent(cursor_id: &str) -> anyhow::Result<()> {+ CONNS.with(|c| {+ let mut b = c.borrow_mut();+ let (_conn, queue) = b.get_mut(cursor_id)+ .ok_or_else(|| anyhow::anyhow!("no persistent connection for cursor '{cursor_id}'"))?;+ let mut tmp = super::State::default();+ queue.roundtrip(&mut tmp).map_err(|e| anyhow::anyhow!("compositor roundtrip failed: {e}"))?;+ Ok(())+ })+}++// ── public API ────────────────────────────────────────────────────────────++/// Press and HOLD `button` (evdev code) at output coordinates `(x, y)` on the+/// toplevel identified by `window_id`. Subsequent `move_to` / `release` calls+/// targeting the same `cursor_id` reuse the same virtual-pointer device, so+/// the compositor treats the sequence as one logical drag rather than as+/// independent clicks. Errors if `cursor_id` already has a held button.+pub fn press(cursor_id: &str, window_id: u64, x: i32, y: i32, button: u8) -> anyhow::Result<()> {+ let (tx_r, rx_r) = bounded(1);+ tx().send(Cmd::Press {+ cursor_id: cursor_id.to_string(),+ window_id, x, y, button,+ reply: tx_r,+ }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;+ rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?+}++/// Emit motion_absolute on the held cursor's virtual-pointer. Errors if there+/// is no held button for `cursor_id`.+pub fn move_to(cursor_id: &str, x: i32, y: i32) -> anyhow::Result<()> {+ let (tx_r, rx_r) = bounded(1);+ tx().send(Cmd::MoveTo {+ cursor_id: cursor_id.to_string(), x, y, reply: tx_r,+ }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;+ rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?+}++/// Release `button` on the held cursor. If no other buttons remain held the+/// virtual-pointer is destroyed and its Wayland connection torn down.+pub fn release(cursor_id: &str, button: u8) -> anyhow::Result<()> {+ let (tx_r, rx_r) = bounded(1);+ tx().send(Cmd::Release {+ cursor_id: cursor_id.to_string(), button, reply: tx_r,+ }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;+ rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?+}++/// Drop the entry for `cursor_id` without emitting any Wayland events.+/// Useful for recovery — if the agent thinks a button is held but the+/// compositor disagrees, this clears the local state without trying to+/// send a release that would error.+pub fn forget(cursor_id: &str) -> anyhow::Result<()> {+ let (tx_r, rx_r) = bounded(1);+ tx().send(Cmd::Forget {+ cursor_id: cursor_id.to_string(), reply: tx_r,+ }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;+ rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rsnew file mode 100644index 0000000000..cba0dd6c00--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rs@@ -0,0 +1,444 @@+//! Per-window capture via `xdg-desktop-portal` ScreenCast + PipeWire.+//!+//! Reaches GNOME/Mutter and KDE/KWin per-window selection — the+//! cross-DE complement to wlroots `ext-image-copy-capture-v1`. Sequence:+//!+//! 1. ashpd ScreenCast::create_session.+//! 2. select_sources with types=Window|Monitor, cursor_mode=Embedded,+//! persist_mode=Application (consent dialog fires ONCE per process,+//! cached for the lifetime of the requesting binary).+//! 3. start(session, parent_window) — user picks the source window in+//! the portal's system dialog the first time; subsequent calls within+//! the same process reuse the cached choice.+//! 4. open_pipe_wire_remote returns a Unix file descriptor; Streams+//! carries the published node ids.+//! 5. Connect a pipewire client to the fd, open a Stream targeting the+//! advertised node id, negotiate Video/Raw format, dequeue one frame+//! with `STREAM_FLAG_MAP_BUFFERS`, copy pixels, PNG-encode, stop.+//!+//! The session + restore_token are persisted in a process-local+//! `OnceLock<PortalScreencastSession>` so only the first call across the+//! process lifetime triggers the consent dialog; later calls reuse the+//! same PipeWire node id.++use std::os::fd::AsRawFd;+use std::sync::OnceLock;++use ashpd::desktop::screencast::{+ CursorMode, OpenPipeWireRemoteOptions, Screencast,+ SelectSourcesOptions, SourceType, StartCastOptions, Streams,+};+use ashpd::desktop::{CreateSessionOptions, PersistMode, Session};+use ashpd::enumflags2::BitFlags;++/// A live portal ScreenCast session held across calls. Expensive to+/// create (system consent dialog) and cheap to reuse — the OnceLock+/// keeps it alive until the requesting process exits.+struct PortalScreencastSession {+ _session: Session<Screencast>,+ streams: Streams,+ fd: std::os::fd::OwnedFd,+}++unsafe impl Send for PortalScreencastSession {}+unsafe impl Sync for PortalScreencastSession {}++static SESSION: OnceLock<PortalScreencastSession> = OnceLock::new();++/// Capture the user-selected window via `xdg-desktop-portal` ScreenCast.+/// On first invocation per process the portal dialog asks the user to+/// pick a window or monitor; subsequent calls reuse the same node+/// transparently. Returns PNG bytes of the latest frame on the stream.+pub fn screenshot_window_via_portal() -> anyhow::Result<Vec<u8>> {+ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?;++ if SESSION.get().is_none() {+ let sess = rt.block_on(create_portal_session())?;+ let _ = SESSION.set(sess);+ }+ let session = SESSION+ .get()+ .ok_or_else(|| anyhow::anyhow!("portal ScreenCast session unavailable"))?;++ let node_id = session+ .streams+ .streams()+ .iter()+ .next()+ .map(|s| s.pipe_wire_node_id())+ .ok_or_else(|| anyhow::anyhow!("portal advertised no streams"))?;++ capture_one_frame(session.fd.as_raw_fd(), node_id)+}++async fn create_portal_session() -> anyhow::Result<PortalScreencastSession> {+ let proxy = Screencast::new()+ .await+ .map_err(|e| anyhow::anyhow!("portal ScreenCast proxy unreachable: {e}. Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde."))?;++ let session = proxy+ .create_session(CreateSessionOptions::default())+ .await+ .map_err(|e| anyhow::anyhow!("portal create_session failed: {e}"))?;++ let select_opts = SelectSourcesOptions::default()+ .set_sources(BitFlags::<SourceType>::from(SourceType::Window) | SourceType::Monitor)+ .set_cursor_mode(CursorMode::Embedded)+ .set_persist_mode(PersistMode::Application);++ proxy+ .select_sources(&session, select_opts)+ .await+ .map_err(|e| anyhow::anyhow!("portal select_sources failed: {e}"))?+ .response()+ .map_err(|e| anyhow::anyhow!("portal select_sources response error: {e}"))?;++ let streams = proxy+ .start(&session, None, StartCastOptions::default())+ .await+ .map_err(|e| anyhow::anyhow!("portal start failed: {e}"))?+ .response()+ .map_err(|e| anyhow::anyhow!("portal start response error (user denied?): {e}"))?;++ let fd = proxy+ .open_pipe_wire_remote(&session, OpenPipeWireRemoteOptions::default())+ .await+ .map_err(|e| anyhow::anyhow!("portal open_pipe_wire_remote failed: {e}"))?;++ Ok(PortalScreencastSession {+ _session: session,+ streams,+ fd,+ })+}++/// Pull one frame off the PipeWire stream advertised by the portal,+/// encode it as PNG, and return the bytes.+///+/// Pipeline:+/// 1. `pipewire::MainLoop` + `Context` + `Core::connect_fd(portal_fd)` —+/// join the portal-published PipeWire remote on the same fd ashpd+/// returned from `open_pipe_wire_remote`.+/// 2. `Stream::new` with `media.type=Video, media.category=Capture,+/// media.role=Screen` so PipeWire knows to route the cast properly.+/// 3. Build an `EnumFormat` SPA pod listing the channel-orders we accept+/// (BGRx, BGRA, RGBx, RGBA) at any size up to 8K — the compositor+/// picks one and announces it via `param_changed`. We track w/h/fmt+/// from that callback.+/// 4. The `process` callback dequeues exactly one buffer, copies the+/// pixels out into a shared `Arc<Mutex<...>>`, and quits the main+/// loop. Subsequent buffers are dropped (queued back via Buffer::Drop)+/// once `frame.is_some()`.+/// 5. After `mainloop.run()` returns, channel-swap the captured bytes+/// into RGBA and PNG-encode them via the `image` crate.+fn capture_one_frame(pipewire_fd: i32, node_id: u32) -> anyhow::Result<Vec<u8>> {+ use std::os::fd::FromRawFd;+ use std::sync::{Arc, Mutex};++ use pipewire as pw;+ use pw::{properties::properties, spa};+ use spa::pod::Pod;++ pw::init();++ let mainloop = pw::main_loop::MainLoop::new(None)+ .map_err(|e| anyhow::anyhow!("pipewire MainLoop::new failed: {e}"))?;+ let context = pw::context::Context::new(&mainloop)+ .map_err(|e| anyhow::anyhow!("pipewire Context::new failed: {e}"))?;++ // dup the portal fd so the OwnedFd we hand to connect_fd has an+ // independent lifecycle from the PortalScreencastSession's master copy.+ let dup_fd = unsafe { libc::dup(pipewire_fd) };+ if dup_fd < 0 {+ anyhow::bail!(+ "dup of portal PipeWire fd failed: {}",+ std::io::Error::last_os_error()+ );+ }+ let owned_fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(dup_fd) };+ let core = context+ .connect_fd(owned_fd, None)+ .map_err(|e| anyhow::anyhow!("pipewire connect_fd failed: {e}"))?;++ // Shared state populated by the param_changed + process callbacks.+ // FrameBuf carries the negotiated geometry alongside the pixel data so+ // the PNG encoder picks the right stride / channel order on exit.+ struct FrameBuf {+ bytes: Vec<u8>,+ width: u32,+ height: u32,+ stride: u32,+ format: spa::param::video::VideoFormat,+ }+ let frame: Arc<Mutex<Option<FrameBuf>>> = Arc::new(Mutex::new(None));+ let neg_format: Arc<Mutex<spa::param::video::VideoInfoRaw>> =+ Arc::new(Mutex::new(Default::default()));++ let stream = pw::stream::Stream::new(+ &core,+ "cua-driver-capture",+ properties! {+ *pw::keys::MEDIA_TYPE => "Video",+ *pw::keys::MEDIA_CATEGORY => "Capture",+ *pw::keys::MEDIA_ROLE => "Screen",+ },+ )+ .map_err(|e| anyhow::anyhow!("pipewire Stream::new failed: {e}"))?;++ let mainloop_for_process = mainloop.clone();+ let neg_format_for_param = neg_format.clone();+ let frame_for_process = frame.clone();+ let neg_format_for_process = neg_format.clone();++ let _listener = stream+ .add_local_listener::<()>()+ .param_changed(move |_stream, _user, id, param| {+ // The compositor confirms the negotiated VideoInfoRaw via the+ // Format param event right after stream.connect — capture it+ // so the process callback knows the width/height/format.+ let Some(param) = param else { return };+ if id != spa::param::ParamType::Format.as_raw() {+ return;+ }+ let (media_type, media_subtype) = match spa::param::format_utils::parse_format(param) {+ Ok(v) => v,+ Err(_) => return,+ };+ if media_type != spa::param::format::MediaType::Video+ || media_subtype != spa::param::format::MediaSubtype::Raw+ {+ return;+ }+ let mut info = spa::param::video::VideoInfoRaw::default();+ if info.parse(param).is_ok() {+ if let Ok(mut g) = neg_format_for_param.lock() {+ *g = info;+ }+ }+ })+ .process(move |stream, _user| {+ // Already captured? Drop further buffers (they'll be queued+ // back via Buffer::Drop) so the run loop can finish.+ if frame_for_process.lock().ok().map(|g| g.is_some()).unwrap_or(false) {+ let _ = stream.dequeue_buffer();+ return;+ }+ let mut buffer = match stream.dequeue_buffer() {+ Some(b) => b,+ None => return,+ };+ let datas = buffer.datas_mut();+ if datas.is_empty() {+ return;+ }+ let info = match neg_format_for_process.lock() {+ Ok(g) => *g,+ Err(_) => return,+ };+ let size_struct = info.size();+ let width = size_struct.width;+ let height = size_struct.height;+ if width == 0 || height == 0 {+ return;+ }+ let chunk_stride = datas[0].chunk().stride();+ let chunk_size = datas[0].chunk().size();+ // PipeWire sometimes leaves stride at 0 (when the producer+ // doesn't fill it in); fall back to width*4 for the packed+ // BGRx/BGRA/RGBx/RGBA formats we negotiate.+ let stride = if chunk_stride > 0 {+ chunk_stride as u32+ } else if chunk_size > 0 && height > 0 {+ chunk_size / height+ } else {+ width * 4+ };+ let payload = match datas[0].data() {+ Some(p) => p,+ None => return,+ };+ let payload_len = (stride as usize) * (height as usize);+ if payload.len() < payload_len {+ return;+ }+ let bytes = payload[..payload_len].to_vec();+ if let Ok(mut g) = frame_for_process.lock() {+ *g = Some(FrameBuf {+ bytes,+ width,+ height,+ stride,+ format: info.format(),+ });+ }+ // Quit the main loop so the caller can encode + return.+ mainloop_for_process.quit();+ })+ .register()+ .map_err(|e| anyhow::anyhow!("pipewire stream listener register failed: {e}"))?;++ // Build an EnumFormat SPA pod listing the formats + sizes we accept.+ // Compositors typically pick BGRx/BGRA on Linux desktops; we list a+ // few alternates so negotiation succeeds on more sources.+ let obj = pw::spa::pod::object!(+ pw::spa::utils::SpaTypes::ObjectParamFormat,+ pw::spa::param::ParamType::EnumFormat,+ pw::spa::pod::property!(+ pw::spa::param::format::FormatProperties::MediaType,+ Id,+ pw::spa::param::format::MediaType::Video+ ),+ pw::spa::pod::property!(+ pw::spa::param::format::FormatProperties::MediaSubtype,+ Id,+ pw::spa::param::format::MediaSubtype::Raw+ ),+ pw::spa::pod::property!(+ pw::spa::param::format::FormatProperties::VideoFormat,+ Choice,+ Enum,+ Id,+ pw::spa::param::video::VideoFormat::BGRx,+ pw::spa::param::video::VideoFormat::BGRx,+ pw::spa::param::video::VideoFormat::BGRA,+ pw::spa::param::video::VideoFormat::RGBx,+ pw::spa::param::video::VideoFormat::RGBA,+ ),+ pw::spa::pod::property!(+ pw::spa::param::format::FormatProperties::VideoSize,+ Choice,+ Range,+ Rectangle,+ pw::spa::utils::Rectangle { width: 1920, height: 1080 },+ pw::spa::utils::Rectangle { width: 1, height: 1 },+ pw::spa::utils::Rectangle { width: 7680, height: 4320 }+ ),+ pw::spa::pod::property!(+ pw::spa::param::format::FormatProperties::VideoFramerate,+ Choice,+ Range,+ Fraction,+ pw::spa::utils::Fraction { num: 30, denom: 1 },+ pw::spa::utils::Fraction { num: 0, denom: 1 },+ pw::spa::utils::Fraction { num: 1000, denom: 1 }+ ),+ );+ let values: Vec<u8> = pw::spa::pod::serialize::PodSerializer::serialize(+ std::io::Cursor::new(Vec::new()),+ &pw::spa::pod::Value::Object(obj),+ )+ .map_err(|e| anyhow::anyhow!("EnumFormat pod serialization failed: {e}"))?+ .0+ .into_inner();+ let mut params = [Pod::from_bytes(&values)+ .ok_or_else(|| anyhow::anyhow!("EnumFormat pod parse-back failed"))?];++ stream+ .connect(+ spa::utils::Direction::Input,+ Some(node_id),+ pw::stream::StreamFlags::AUTOCONNECT | pw::stream::StreamFlags::MAP_BUFFERS,+ &mut params,+ )+ .map_err(|e| anyhow::anyhow!("pipewire stream.connect failed: {e}"))?;++ // Install a 10-second timeout source so a producer that never delivers+ // a frame (eg the user closed the consent dialog mid-cast) doesn't+ // hang the main loop forever.+ let timeout_loop = mainloop.clone();+ let _timeout_source = mainloop+ .loop_()+ .add_timer(move |_expirations: u64| timeout_loop.quit());+ _timeout_source+ .update_timer(Some(std::time::Duration::from_secs(10)), None)+ .into_result()+ .map_err(|e| anyhow::anyhow!("pipewire timeout source failed: {e:?}"))?;++ mainloop.run();++ // Tear down the stream before we read the frame, so the producer+ // can't keep mutating the buffer underneath us.+ drop(_listener);+ drop(stream);++ let frame = frame+ .lock()+ .map_err(|_| anyhow::anyhow!("frame mutex poisoned"))?+ .take()+ .ok_or_else(|| anyhow::anyhow!("portal ScreenCast: no frame arrived within timeout"))?;++ encode_frame_to_png(&frame.bytes, frame.width, frame.height, frame.stride, frame.format)+}++/// Channel-swap a PipeWire video frame into RGBA8888 and PNG-encode it.+///+/// Mirrors `ext_screencopy::encode_buffer_to_png` — the input layout+/// depends on the negotiated VideoFormat: BGRx/BGRA are little-endian+/// BGRA (so swap R↔B), RGBx/RGBA are already RGBA in memory.+fn encode_frame_to_png(+ pixels: &[u8],+ width: u32,+ height: u32,+ stride: u32,+ fmt: libspa::param::video::VideoFormat,+) -> anyhow::Result<Vec<u8>> {+ use libspa::param::video::VideoFormat;+ let mut rgba: Vec<u8> = Vec::with_capacity((width as usize) * (height as usize) * 4);+ for y in 0..(height as usize) {+ let row_start = y * (stride as usize);+ for x in 0..(width as usize) {+ let i = row_start + x * 4;+ let (r, g, b, a) = match fmt {+ VideoFormat::BGRx => (pixels[i + 2], pixels[i + 1], pixels[i], 0xFF),+ VideoFormat::BGRA => (pixels[i + 2], pixels[i + 1], pixels[i], pixels[i + 3]),+ VideoFormat::RGBx => (pixels[i], pixels[i + 1], pixels[i + 2], 0xFF),+ VideoFormat::RGBA => (pixels[i], pixels[i + 1], pixels[i + 2], pixels[i + 3]),+ // Fall through: treat unknown formats as BGRA (the most+ // common compositor output on Linux).+ _ => (pixels[i + 2], pixels[i + 1], pixels[i], pixels[i + 3]),+ };+ rgba.extend_from_slice(&[r, g, b, a]);+ }+ }+ use image::{codecs::png::PngEncoder, ImageBuffer, ImageEncoder, Rgba};+ let img: ImageBuffer<Rgba<u8>, _> =+ ImageBuffer::from_raw(width, height, rgba).ok_or_else(|| {+ anyhow::anyhow!("internal: buffer dims mismatch ({}x{})", width, height)+ })?;+ let mut out: Vec<u8> = Vec::new();+ PngEncoder::new(&mut out).write_image(+ img.as_raw(),+ width,+ height,+ image::ExtendedColorType::Rgba8,+ )?;+ Ok(out)+}++/// Probe whether the portal ScreenCast interface is reachable on the+/// session bus without taking an actual capture (avoids cached consent).+pub fn probe_screencast_portal() -> anyhow::Result<bool> {+ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime: {e}"))?;+ rt.block_on(async {+ let conn = zbus::Connection::session()+ .await+ .map_err(|e| anyhow::anyhow!("session bus unreachable: {e}"))?;+ let proxy = zbus::fdo::DBusProxy::new(&conn)+ .await+ .map_err(|e| anyhow::anyhow!("dbus proxy creation failed: {e}"))?;+ let bus_name: zbus::names::BusName = "org.freedesktop.portal.Desktop"+ .try_into()+ .map_err(|e| anyhow::anyhow!("bus name parse failed: {e}"))?;+ let has_owner = proxy+ .name_has_owner(bus_name)+ .await+ .map_err(|e| anyhow::anyhow!("name_has_owner failed: {e}"))?;+ Ok(has_owner)+ })+}diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rsnew file mode 100644index 0000000000..c1f1afbc0a--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs@@ -0,0 +1,124 @@+//! `xdg-desktop-portal` Screenshot capture path.+//!+//! Reaches GNOME/Mutter, KDE/KWin, and any compositor that ships an+//! xdg-desktop-portal backend — superset of the wlroots-only+//! `zwlr_screencopy_manager_v1` path. The portal asks the user for consent+//! on first invocation per session (GNOME caches via `Settings → Privacy →+//! Screenshot`; KDE Plasma 6 caches per requesting binary); subsequent+//! calls in the same session are silent.+//!+//! ashpd is async (zbus under the hood); the rest of `wayland::screenshot_*`+//! is sync because it's called from `tokio::task::spawn_blocking`. We build+//! a dedicated single-threaded tokio runtime per call so the async ashpd+//! work has somewhere to live without colliding with the caller's runtime.++use std::path::PathBuf;++use ashpd::desktop::screenshot::Screenshot;++/// Capture the current display via `org.freedesktop.portal.Screenshot.Screenshot`.+/// Returns the PNG/JPEG bytes the portal wrote to disk, or a typed error+/// when the portal isn't reachable (e.g. wlroots compositor without+/// `xdg-desktop-portal-wlr`, KDE without `xdg-desktop-portal-kde` running,+/// or no D-Bus session bus).+///+/// `interactive=false` skips the customization dialog (region select,+/// post-shot editor) — the user still consents to the screenshot itself+/// on the first call per session unless they've pre-approved the+/// requesting binary in their DE's privacy settings.+pub fn screenshot_via_portal() -> anyhow::Result<Vec<u8>> {+ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?;+ let uri = rt.block_on(async {+ Screenshot::request()+ .interactive(false)+ .modal(false)+ .send()+ .await+ .map_err(map_ashpd_err)?+ .response()+ .map_err(map_ashpd_err)+ .map(|resp| resp.uri().clone())+ })?;++ // The portal returns a file:// URI pointing at a PNG (typically in+ // /tmp or $XDG_RUNTIME_DIR/doc/...). Decode it via the `url` crate.+ let path = uri_to_path(&uri.to_string())+ .ok_or_else(|| anyhow::anyhow!("portal returned a non-file URI: {uri}"))?;+ let bytes = std::fs::read(&path)+ .map_err(|e| anyhow::anyhow!("portal screenshot at {} unreadable: {e}", path.display()))?;+ // Best-effort cleanup: the portal places the file in a process-readable+ // location that can outlive our process. Removing avoids the disk+ // accumulating one PNG per call. If removal fails (permissions, file+ // already gone) we silently drop the error — the bytes are already+ // captured.+ let _ = std::fs::remove_file(&path);+ Ok(bytes)+}++/// Convert a `file://...` URI string to a local PathBuf. Returns None for+/// non-file schemes (e.g. portal `doc://...` URIs in some sandboxed+/// configurations, which would need a second resolution step through the+/// document portal).+fn uri_to_path(uri_str: &str) -> Option<PathBuf> {+ let url = url::Url::parse(uri_str).ok()?;+ if url.scheme() != "file" {+ return None;+ }+ url.to_file_path().ok()+}++fn map_ashpd_err<E: std::fmt::Display>(e: E) -> anyhow::Error {+ let msg = format!("xdg-desktop-portal Screenshot failed: {e}");+ // Pattern-match common failure modes to produce a typed-ish error+ // the caller can pivot on.+ if msg.contains("ServiceUnknown") || msg.contains("NotFound") {+ anyhow::anyhow!("xdg-desktop-portal is not running on this session ({msg}). Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde / xdg-desktop-portal-wlr for your compositor.")+ } else if msg.contains("Cancelled") || msg.contains("denied") {+ anyhow::anyhow!("xdg-desktop-portal Screenshot consent dialog was cancelled or denied: {msg}")+ } else {+ anyhow::anyhow!(msg)+ }+}++/// Probe whether the portal *service* is reachable on the session bus.+/// Returns Ok(true) when the `org.freedesktop.portal.Desktop` name has an+/// owner, Ok(false) otherwise, and Err(...) for transport failures+/// (no session bus, etc.).+///+/// Read-only and side-effect free — does NOT actually invoke the+/// Screenshot interface, so it doesn't cache a consent grant on GNOME /+/// KDE backends that gate Screenshot on user prompts. The doctor uses+/// this to surface portal availability without burning an interactive+/// dialog every time `hermes computer-use doctor` runs.+///+/// Note: a true return here means the portal *service* is present. The+/// Screenshot *interface* on that service may still be missing or+/// disabled — for that, an actual `Screenshot::request().send().await`+/// is needed. We accept the false-positive risk in exchange for the+/// no-consent, no-burn-dialog property.+pub fn probe_portal() -> anyhow::Result<bool> {+ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd probe: {e}"))?;+ rt.block_on(async {+ // Cheapest possible probe: open a session bus and check that the+ // portal name has an owner. We avoid Screenshot::request() because+ // that would cache a (potentially undesired) consent grant.+ let conn = zbus::Connection::session()+ .await+ .map_err(|e| anyhow::anyhow!("session bus unreachable: {e}"))?;+ let proxy = zbus::fdo::DBusProxy::new(&conn)+ .await+ .map_err(|e| anyhow::anyhow!("dbus proxy creation failed: {e}"))?;+ let has_owner = proxy+ .name_has_owner("org.freedesktop.portal.Desktop".try_into()+ .map_err(|e| anyhow::anyhow!("bus name parse failed: {e}"))?)+ .await+ .map_err(|e| anyhow::anyhow!("name_has_owner failed: {e}"))?;+ Ok(has_owner)+ })+}diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nixindex 30cb7e39f9..409f163c86 100644--- a/nix/cua-driver/package.nix+++ b/nix/cua-driver/package.nix@@ -46,14 +46,31 @@ pkgs.rustPlatform.buildRustPackage {# ureq -> rustls (no openssl)# tiny-skia -> pure Rust 2D graphics# ring -> compiles own C/asm via stdenv's cc- # Except the `x11` crate (raw Xlib FFI for MPX multi-cursor drags), whose- # build.rs locates libX11/libXi/libXtst via pkg-config.- nativeBuildInputs = [ pkgs.pkg-config ];+ # The `x11` crate (raw Xlib FFI for MPX multi-cursor drags) needs+ # libX11/libXi/libXtst via pkg-config. The Wayland-parity work+ # (round 2/3 — wayland::portal_screencast / wayland::libei) adds:+ # pipewire 0.8 -> needs libpipewire-0.3 + libspa headers via pkg-config+ # + bindgen (clang) for SPA pod generation+ # reis 0.7 -> pure Rust libei binding, but the workspace pulls+ # libei-dev transitively through ashpd's tokio feature+ nativeBuildInputs = with pkgs; [+ pkg-config+ # bindgen needs a libclang at build time for the libpipewire / libspa+ # SPA pod codegen. rust-bindgen also picks up `stdbool.h` etc. from+ # clang's resource dir, not glibc.+ rustPlatform.bindgenHook+ ];buildInputs = with pkgs; [libx11libxilibxtstlibxext+ # Wayland-parity additions: PipeWire is needed by the portal+ # ScreenCast capture path (wayland::portal_screencast). pipewire+ # already pulls libspa transitively in nixpkgs.+ pipewire+ # libei via reis — same as the ashpd RemoteDesktop+EIS flow.+ libei];# Skip tests that require a running X11 display or AT-SPI bus