Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(platform-linux): native-Wayland parity (wlroots + portal + libei) (#1966)

Francesco Bonacci519dfdfbd6c7
diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock
index 362556d4dd..c0805d8e90 100644
--- a/libs/cua-driver/rust/Cargo.lock
+++ b/libs/cua-driver/rust/Cargo.lock
@@ -17,6 +17,16 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "annotate-snippets"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccaf7e9dfbb6ab22c82e473cd1a8a7bd313c19a5b7e40970f3d89ef5a5c9e81e"
+dependencies = [
+ "unicode-width",
+ "yansi-term",
+]
+
[[package]]
name = "anyhow"
version = "1.0.102"
@@ -53,6 +63,21 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+[[package]]
+name = "ashpd"
+version = "0.13.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "340e0f6bf7f9ee78549c61454f1460a3ed97c011902ee76b58301bbc6d502a32"
+dependencies = [
+ "enumflags2",
+ "futures-util",
+ "getrandom 0.4.2",
+ "serde",
+ "serde_repr",
+ "tokio",
+ "zbus",
+]
+
[[package]]
name = "async-broadcast"
version = "0.7.2"
@@ -253,6 +278,27 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "bindgen"
+version = "0.69.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
+dependencies = [
+ "annotate-snippets",
+ "bitflags 2.11.1",
+ "cexpr",
+ "clang-sys",
+ "itertools",
+ "lazy_static",
+ "lazycell",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn",
+]
+
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -338,6 +384,19 @@ version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
+[[package]]
+name = "calloop"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7"
+dependencies = [
+ "bitflags 2.11.1",
+ "polling",
+ "rustix",
+ "slab",
+ "tracing",
+]
+
[[package]]
name = "cc"
version = "1.2.62"
@@ -348,12 +407,42 @@ dependencies = [
"shlex",
]
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom 7.1.3",
+]
+
+[[package]]
+name = "cfg-expr"
+version = "0.15.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02"
+dependencies = [
+ "smallvec",
+ "target-lexicon",
+]
+
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@@ -363,6 +452,15 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "convert_case"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca"
+dependencies = [
+ "unicode-segmentation",
+]
+
[[package]]
name = "cookie"
version = "0.18.1"
@@ -374,6 +472,15 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "cookie-factory"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2"
+dependencies = [
+ "futures",
+]
+
[[package]]
name = "cookie_store"
version = "0.22.1"
@@ -459,6 +566,15 @@ dependencies = [
"cfg-if",
]
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
+dependencies = [
+ "crossbeam-utils",
+]
+
[[package]]
name = "crossbeam-queue"
version = "0.3.12"
@@ -590,6 +706,27 @@ dependencies = [
"crypto-common",
]
+[[package]]
+name = "dirs"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
+dependencies = [
+ "libc",
+ "option-ext",
+ "redox_users",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "displaydoc"
version = "0.2.5"
@@ -626,6 +763,12 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
+[[package]]
+name = "either"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
+
[[package]]
name = "embed-manifest"
version = "1.5.0"
@@ -707,7 +850,7 @@ dependencies = [
"bitvec",
"cfg-if",
"libc",
- "nix",
+ "nix 0.23.2",
"thiserror",
]
@@ -867,12 +1010,48 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
+[[package]]
+name = "futures"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
[[package]]
name = "futures-core"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+[[package]]
+name = "futures-executor"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
[[package]]
name = "futures-io"
version = "0.3.32"
@@ -921,10 +1100,13 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
dependencies = [
+ "futures-channel",
"futures-core",
+ "futures-io",
"futures-macro",
"futures-sink",
"futures-task",
+ "memchr",
"pin-project-lite",
"slab",
]
@@ -973,6 +1155,12 @@ dependencies = [
"wasip3",
]
+[[package]]
+name = "glob"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+
[[package]]
name = "hashbrown"
version = "0.15.5"
@@ -1164,6 +1352,15 @@ dependencies = [
"serde_core",
]
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.18"
@@ -1199,6 +1396,12 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
[[package]]
name = "leb128fmt"
version = "0.1.0"
@@ -1211,12 +1414,59 @@ version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+[[package]]
+name = "libloading"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
+dependencies = [
+ "cfg-if",
+ "windows-link 0.2.1",
+]
+
[[package]]
name = "libm"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
+[[package]]
+name = "libredox"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "libspa"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65f3a4b81b2a2d8c7f300643676202debd1b7c929dbf5c9bb89402ea11d19810"
+dependencies = [
+ "bitflags 2.11.1",
+ "cc",
+ "convert_case",
+ "cookie-factory",
+ "libc",
+ "libspa-sys",
+ "nix 0.27.1",
+ "nom 7.1.3",
+ "system-deps",
+]
+
+[[package]]
+name = "libspa-sys"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf0d9716420364790e85cbb9d3ac2c950bde16a7dd36f3209b7dfdfc4a24d01f"
+dependencies = [
+ "bindgen",
+ "cc",
+ "system-deps",
+]
+
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
@@ -1265,6 +1515,15 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+[[package]]
+name = "memmap2"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "memoffset"
version = "0.6.5"
@@ -1290,9 +1549,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f79496a5651c8d57cd033c5add8ca7ee4e3d5f7587a4777484640d9cb60392d9"
dependencies = [
"fnv",
- "nom",
+ "nom 1.2.4",
]
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
[[package]]
name = "miniz_oxide"
version = "0.8.9"
@@ -1354,12 +1619,33 @@ dependencies = [
"memoffset 0.6.5",
]
+[[package]]
+name = "nix"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
+dependencies = [
+ "bitflags 2.11.1",
+ "cfg-if",
+ "libc",
+]
+
[[package]]
name = "nom"
version = "1.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5b8c256fd9471521bcb84c3cdba98921497f1a331cbc15b8030fc63b82050ce"
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
@@ -1532,6 +1818,12 @@ dependencies = [
"vcpkg",
]
+[[package]]
+name = "option-ext"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
+
[[package]]
name = "ordered-stream"
version = "0.2.0"
@@ -1609,6 +1901,34 @@ dependencies = [
"futures-io",
]
+[[package]]
+name = "pipewire"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08e645ba5c45109106d56610b3ee60eb13a6f2beb8b74f8dc8186cf261788dda"
+dependencies = [
+ "anyhow",
+ "bitflags 2.11.1",
+ "libc",
+ "libspa",
+ "libspa-sys",
+ "nix 0.27.1",
+ "once_cell",
+ "pipewire-sys",
+ "thiserror",
+]
+
+[[package]]
+name = "pipewire-sys"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "849e188f90b1dda88fe2bfe1ad31fe5f158af2c98f80fb5d13726c44f3f01112"
+dependencies = [
+ "bindgen",
+ "libspa-sys",
+ "system-deps",
+]
+
[[package]]
name = "pkg-config"
version = "0.3.33"
@@ -1620,26 +1940,39 @@ name = "platform-linux"
version = "0.6.0"
dependencies = [
"anyhow",
+ "ashpd",
"async-trait",
"atspi",
"base64",
+ "calloop",
+ "crossbeam-channel",
"cua-driver-core",
"cursor-overlay",
+ "dirs",
+ "enumflags2",
"evdev",
"image",
"libc",
+ "libspa",
"meval",
"pip-preview",
+ "pipewire",
+ "reis",
"serde",
"serde_json",
"thiserror",
"tiny-skia",
"tokio",
"tracing",
+ "tracing-subscriber",
+ "url",
"wayland-client",
+ "wayland-protocols",
"wayland-protocols-wlr",
"x11",
"x11rb",
+ "xkbcommon",
+ "zbus",
]
[[package]]
@@ -1864,6 +2197,29 @@ dependencies = [
"bitflags 2.11.1",
]
+[[package]]
+name = "redox_users"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
+dependencies = [
+ "getrandom 0.2.17",
+ "libredox",
+ "thiserror",
+]
+
+[[package]]
+name = "regex"
+version = "1.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
[[package]]
name = "regex-automata"
version = "0.4.14"
@@ -1881,6 +2237,19 @@ version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+[[package]]
+name = "reis"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81f3fedd2777cde52c1be5e572efbec485eac7b801c47820eda388d4f13b9c4b"
+dependencies = [
+ "enumflags2",
+ "futures-util",
+ "log",
+ "rustix",
+ "tokio",
+]
+
[[package]]
name = "resvg"
version = "0.44.0"
@@ -1924,6 +2293,12 @@ version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97"
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
[[package]]
name = "rustc_version"
version = "0.4.1"
@@ -2269,6 +2644,19 @@ dependencies = [
"syn",
]
+[[package]]
+name = "system-deps"
+version = "6.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349"
+dependencies = [
+ "cfg-expr",
+ "heck",
+ "pkg-config",
+ "toml",
+ "version-compare",
+]
+
[[package]]
name = "tap"
version = "1.0.1"
@@ -2285,6 +2673,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
[[package]]
name = "tempfile"
version = "3.27.0"
@@ -2539,6 +2933,7 @@ version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
+ "log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
@@ -2675,12 +3070,24 @@ version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "383ad40bb927465ec0ce7720e033cb4ca06912855fc35db31b5755d0de75b1ee"
+[[package]]
+name = "unicode-segmentation"
+version = "1.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
+
[[package]]
name = "unicode-vo"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1d386ff53b415b7fe27b50bb44679e2cc4660272694b7b6f3326d8480823a94"
+[[package]]
+name = "unicode-width"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
+
[[package]]
name = "unicode-xid"
version = "0.2.6"
@@ -2806,6 +3213,12 @@ version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+[[package]]
+name = "version-compare"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e"
+
[[package]]
name = "version_check"
version = "0.9.5"
@@ -3023,6 +3436,28 @@ dependencies = [
"rustls-pki-types",
]
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
[[package]]
name = "windows"
version = "0.58.0"
@@ -3516,12 +3951,38 @@ version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd"
+[[package]]
+name = "xkbcommon"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7a974f48060a14e95705c01f24ad9c3345022f4d97441b8a36beb7ed5c4a02d"
+dependencies = [
+ "libc",
+ "memmap2",
+ "xkeysym",
+]
+
+[[package]]
+name = "xkeysym"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56"
+
[[package]]
name = "xmlwriter"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9"
+[[package]]
+name = "yansi-term"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe5c30ade05e61656247b2e334a031dfd0cc466fadef865bdcdea8d537951bf1"
+dependencies = [
+ "winapi",
+]
+
[[package]]
name = "yoke"
version = "0.8.2"
diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml
index 77515608c1..0138e3fca1 100644
--- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml
+++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml
@@ -38,3 +38,68 @@ meval = "0.2"
# when running under a Wayland compositor with no X11 (no DISPLAY).
wayland-client = "0.31"
wayland-protocols-wlr = { version = "0.3", features = ["client"] }
+# Wayland staging protocols: ext-foreign-toplevel-list-v1,
+# ext-image-capture-source-v1, ext-image-copy-capture-v1 (per-window
+# screenshot on modern wlroots/GNOME/KDE) and wp-fractional-scale-v1 +
+# wp-viewporter (HiDPI for the layer-shell overlay). The transitive pull
+# from wayland-protocols-wlr does NOT enable `staging`; declaring it here
+# unifies the feature globally.
+wayland-protocols = { version = "0.32", features = ["client", "staging"] }
+# xdg-desktop-portal client: display-screenshot tier
+# (org.freedesktop.portal.Screenshot), per-window ScreenCast
+# (org.freedesktop.portal.ScreenCast), and the libei input session
+# (org.freedesktop.portal.RemoteDesktop.ConnectToEIS). `tokio` shares the
+# driver's async runtime.
+ashpd = { version = "0.13", features = ["tokio", "screencast", "remote_desktop", "screenshot"] }
+# zbus is pulled transitively via ashpd + atspi but declared directly so
+# the portal-probe path in wayland::portal_screenshot can talk to the
+# session bus without going through ashpd's higher-level builders.
+zbus = { version = "5", default-features = false, features = ["tokio"] }
+# Parse `file://` URIs returned by org.freedesktop.portal.Screenshot.
+# ashpd re-exports `url` transitively; declared explicitly so ashpd
+# version bumps cannot silently break the resolver.
+url = "2"
+# PipeWire client for the ScreenCast per-window capture path — dequeues
+# frames from the node id returned by `open_pipe_wire_remote`. Pinned to
+# 0.8 (last well-documented release; 0.10 had docs.rs build failures in
+# mid-2026). Needs libpipewire-0.3 headers at build time.
+pipewire = "0.8"
+# Companion to pipewire 0.8 — needed to construct the SPA_PARAM_EnumFormat
+# pod (SPA_VIDEO_FORMAT_BGRx at announced width/height) we hand to
+# `Stream::connect`.
+libspa = "0.8"
+# Pure-Rust libei/libeis bindings for the GNOME/KDE input path (handshake
+# → seat → device → pointer_absolute/button/scroll/keyboard/text). API is
+# documented as "incomplete and subject to change" — budget for churn on
+# the next bump. Re-exports `enumflags2` we use for `BitFlags<DeviceType>`.
+reis = { version = "0.7", features = ["tokio"] }
+# Calloop event loop drives the EIS worker thread — reis's example uses
+# calloop::EventLoop + Generic source to multiplex EIS protocol reads
+# with command-channel polling. reis 0.7 also ships a calloop adapter
+# but we use the plain Generic source for clarity.
+calloop = "0.14"
+# Used directly in the ashpd RemoteDesktop call to compose
+# `BitFlags::from(DeviceType::Keyboard) | DeviceType::Pointer`. reis also
+# re-exports this same major version; pinning explicitly prevents
+# duplicate-crate hazards.
+enumflags2 = "0.7"
+# Keymap fallback path for libei keyboard input when the EIS server does
+# NOT advertise `ei_text` (libei < 1.6, i.e. older GNOME / KDE Plasma 6.0).
+# Mirrors the reis `type-text.rs` example's keycode reverse-mapping.
+xkbcommon = "0.9"
+# Bounded MPSC command/reply channel for the persistent virtual-pointer
+# owner thread. EventQueue<State> is !Send, so the persistent device must
+# live on its own thread; crossbeam gives us a Send sender and synchronous
+# reply channels that preserve the "tool call blocks until compositor
+# roundtripped" contract the X11 path provides.
+crossbeam-channel = "0.5"
+# Resolve `$XDG_STATE_HOME` (with platform fallback) for persisting the
+# ScreenCast and RemoteDesktop `restore_token`s so users see the consent
+# dialog only once per install, not once per process.
+dirs = "5"
+
+[dev-dependencies]
+# Used by the `screenshot_cascade` example to surface the
+# `tracing::debug!` lines emitted by each tier of the dispatch as it
+# falls through.
+tracing-subscriber = { workspace = true }
diff --git a/libs/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rs b/libs/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rs
new file mode 100644
index 0000000000..bf0de170e8
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/examples/screenshot_cascade.rs
@@ -0,0 +1,52 @@
+//! Standalone smoke for the platform-linux screenshot cascade.
+//!
+//! Calls `capture::screenshot_display_bytes()` and prints the PNG size +
+//! magic bytes + which tier produced the result (inferred from
+//! tracing-subscriber logs).
+//!
+//! Usage:
+//! CUA_DRIVER_RS_ENABLE_WAYLAND=1 \
+//! WAYLAND_DISPLAY=wayland-0 \
+//! cargo run --example screenshot_cascade --release
+
+use std::io::Write as _;
+
+fn main() {
+ // Bare tracing subscriber so the debug! lines from the cascade tiers
+ // show up.
+ let _ = tracing_subscriber::FmtSubscriber::builder()
+ .with_max_level(tracing::Level::DEBUG)
+ .with_writer(std::io::stderr)
+ .try_init();
+
+ eprintln!("== platform-linux screenshot cascade smoke ==");
+ eprintln!(
+ " WAYLAND_DISPLAY={:?} DISPLAY={:?} CUA_DRIVER_RS_ENABLE_WAYLAND={:?}",
+ std::env::var("WAYLAND_DISPLAY").ok(),
+ std::env::var("DISPLAY").ok(),
+ std::env::var("CUA_DRIVER_RS_ENABLE_WAYLAND").ok(),
+ );
+
+ let started = std::time::Instant::now();
+ match platform_linux::capture::screenshot_display_bytes() {
+ Ok(bytes) => {
+ let dt = started.elapsed();
+ eprintln!(
+ "OK {} bytes in {dt:?} magic={:?}",
+ bytes.len(),
+ &bytes[..bytes.len().min(8)]
+ );
+ // Write the PNG so we can also visually check it.
+ let path = std::env::var("CUA_SMOKE_OUT")
+ .unwrap_or_else(|_| "/tmp/cua-screenshot.png".to_string());
+ if let Ok(mut f) = std::fs::File::create(&path) {
+ let _ = f.write_all(&bytes);
+ eprintln!(" wrote {}", path);
+ }
+ }
+ Err(e) => {
+ eprintln!("ERR {}", e);
+ std::process::exit(1);
+ }
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
index ccc884637f..589b774b8a 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs
@@ -921,6 +921,15 @@ fn x11_window_origin(xid: u64) -> Option<(i32, i32)> {
/// X11 origin), so the existing behaviour is preserved for non-GTK4 toolkits
/// and the change can never make correct coordinates worse.
async fn gtk4_screen_offset(visited: &[Visited<'_>], pid: u32, xid: u64) -> (i32, i32) {
+ // Native Wayland forbids a client from querying another window's screen
+ // origin (privacy by design), so the X11-based offset recovery below
+ // can't run. Return (0, 0) — GTK4 element bounds will be window-local
+ // rather than screen-relative on Wayland, which mirrors how every other
+ // tool reports coords on this backend.
+ if crate::wayland::is_wayland() {
+ let _ = (visited, pid, xid);
+ return (0, 0);
+ }
// The frame is the toplevel window accessible. Prefer an explicit "frame"
// role; fall back to the first node that exposes a Component interface.
let frame = visited
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs
index 831ae61bb9..28908bb509 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs
@@ -99,7 +99,32 @@ pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> {
// through `cua_driver_core::image_utils::encode_rgba_to_png`.
/// Capture the primary display (root window) as raw PNG bytes.
+///
+/// Dispatch:
+/// - Native Wayland (`CUA_DRIVER_RS_ENABLE_WAYLAND=1` + Wayland session):
+/// routes through [`crate::wayland::screenshot_display_dispatch`] which
+/// cascades wlroots screencopy → ext-image-copy-capture-v1 → portal
+/// Screenshot. Each tier falls through to the next on missing globals.
+/// - X11 / Wayland-disabled: ImageMagick `import` → x11rb `XGetImage`.
pub fn screenshot_display_bytes() -> Result<Vec<u8>> {
+ if crate::wayland::is_wayland() {
+ match crate::wayland::screenshot_display_dispatch() {
+ Ok(bytes) => return Ok(bytes),
+ Err(e) => {
+ tracing::debug!(
+ "wayland screenshot cascade unavailable ({e}); falling through to X11"
+ );
+ }
+ }
+ }
+ screenshot_display_bytes_x11()
+}
+
+/// X11-only display capture path — extracted so the wayland cascade in
+/// [`crate::wayland::screenshot_display_dispatch`] can call it as a final
+/// fallback without re-entering [`screenshot_display_bytes`] (which would
+/// loop forever once we're on Wayland).
+pub(crate) fn screenshot_display_bytes_x11() -> Result<Vec<u8>> {
// Try `import -window root png:-` (ImageMagick).
let out = Command::new("import")
.args(["-window", "root", "png:-"])
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs b/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs
index 03d146de88..0c44bce561 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs
@@ -15,6 +15,11 @@ use cua_driver_core::health_report::{
NAME_SESSION_ACTIVE, NAME_TCC_ACCESSIBILITY, NAME_TCC_SCREEN_RECORDING,
};
+/// Doctor entry that surfaces which wlroots manager globals the running
+/// Wayland compositor advertises (foreign-toplevel / screencopy /
+/// virtual-pointer / wl_shm). Linux-specific; skipped when not on Wayland.
+pub const NAME_WAYLAND_BACKEND: &str = "wayland_backend";
+
/// Linux canonical check names — same Swift-PR contract, with TCC /
/// bundle entries surfaced as `skip("not applicable on Linux")`. The
/// full set of entries always appears so cross-platform consumers see
@@ -28,6 +33,7 @@ pub const LINUX_CHECK_NAMES: &[&str] = &[
NAME_TCC_SCREEN_RECORDING,
NAME_AX_CAPABILITY,
NAME_SCREEN_CAPTURE_CAPABILITY,
+ NAME_WAYLAND_BACKEND,
];
pub struct LinuxHealthProvider;
@@ -52,6 +58,7 @@ impl HealthCheckProvider for LinuxHealthProvider {
NAME_TCC_SCREEN_RECORDING => skip_not_applicable(NAME_TCC_SCREEN_RECORDING),
NAME_AX_CAPABILITY => check_ax_capability().await,
NAME_SCREEN_CAPTURE_CAPABILITY => check_screen_capture_capability().await,
+ NAME_WAYLAND_BACKEND => check_wayland_backend().await,
other => CheckEntry::skip(
other.to_owned(),
"Unknown check name (not implemented on this platform).",
@@ -92,6 +99,28 @@ fn skip_not_applicable(name: &str) -> CheckEntry {
}
async fn check_ax_capability() -> CheckEntry {
+ // Native Wayland: AT-SPI lives entirely on D-Bus, so the AX prereq is
+ // `org.a11y.Bus` being reachable on the session bus. Falls back to X11
+ // reachability for X / XWayland sessions.
+ if is_wayland_session() {
+ let bus_ok = tokio::task::spawn_blocking(probe_a11y_bus)
+ .await
+ .unwrap_or(false);
+ if bus_ok {
+ return CheckEntry::pass(
+ NAME_AX_CAPABILITY,
+ "Wayland session: org.a11y.Bus reachable on the session bus; \
+ AT-SPI inspection will work.",
+ );
+ }
+ return CheckEntry::fail(
+ NAME_AX_CAPABILITY,
+ "Wayland session: org.a11y.Bus is not reachable on the session bus; \
+ AT-SPI inspection will fail.",
+ "Start the AT-SPI bus (`/usr/libexec/at-spi-bus-launcher`) or enable \
+ accessibility in your desktop's settings.",
+ );
+ }
// Mirror the existing `check_permissions` Linux probe: X11
// connectivity is the AX prerequisite (AT-SPI is over D-Bus, but
// input/readback need an X server). Cheap and side-effect-free.
@@ -104,14 +133,76 @@ async fn check_ax_capability() -> CheckEntry {
"X11 reachable; AT-SPI + XSendEvent input will work.",
);
}
+ let hint = if std::env::var_os("WAYLAND_DISPLAY").is_some() {
+ "Pure Wayland session: opt into the experimental Wayland backend by \
+ setting CUA_DRIVER_RS_ENABLE_WAYLAND=1, or run the target under XWayland."
+ } else {
+ "Set DISPLAY (X11) — under Wayland, run via XWayland or expose XDG_SESSION_TYPE=x11."
+ };
CheckEntry::fail(
NAME_AX_CAPABILITY,
"X11 is not reachable; UI inspection and event injection will fail.",
- "Set DISPLAY (X11) — under Wayland, run via XWayland or expose XDG_SESSION_TYPE=x11.",
+ hint,
)
}
async fn check_screen_capture_capability() -> CheckEntry {
+ // Native Wayland: capture flows through a cascade — wlroots screencopy
+ // (sway / labwc / kwin 5.27+ / hyprland) → ext-image-copy-capture-v1
+ // (sway 1.10+ / labwc 0.8+ / niri / KDE 6.2+ / GNOME 47+) →
+ // xdg-desktop-portal Screenshot (GNOME / KDE / COSMIC + portal backend).
+ // Report which tier is reachable so users on mutter / kwin don't see a
+ // misleading "screen capture will fail" just because wlroots isn't
+ // present.
+ if is_wayland_session() {
+ let snap = tokio::task::spawn_blocking(probe_wayland_managers)
+ .await
+ .ok()
+ .and_then(|r| r.ok());
+ match &snap {
+ Some(m) if m.screencopy && m.wl_shm => {
+ return CheckEntry::pass(
+ NAME_SCREEN_CAPTURE_CAPABILITY,
+ "Wayland session: zwlr_screencopy_manager_v1 + wl_shm advertised; \
+ native output capture is functional.",
+ );
+ }
+ Some(m) if m.ext_image_copy_capture && m.ext_output_image_capture_source => {
+ return CheckEntry::pass(
+ NAME_SCREEN_CAPTURE_CAPABILITY,
+ "Wayland session: ext-image-copy-capture-v1 + \
+ ext-output-image-capture-source-v1 advertised; cross-DE \
+ native output capture is functional.",
+ );
+ }
+ _ => {}
+ }
+ // No wlr screencopy AND no ext-image-copy-capture. Probe the
+ // portal as the last native tier. The probe is read-only — it
+ // checks for a name owner on the bus, does NOT take a screenshot.
+ let portal_ok = tokio::task::spawn_blocking(crate::wayland::portal_screenshot::probe_portal)
+ .await
+ .ok()
+ .and_then(|r| r.ok())
+ .unwrap_or(false);
+ if portal_ok {
+ return CheckEntry::pass(
+ NAME_SCREEN_CAPTURE_CAPABILITY,
+ "Wayland session: no native screencopy globals, but \
+ xdg-desktop-portal Screenshot is reachable; capture will go \
+ through the portal (one consent prompt per session).",
+ );
+ }
+ return CheckEntry::fail(
+ NAME_SCREEN_CAPTURE_CAPABILITY,
+ "Wayland session: no native screencopy globals, and \
+ xdg-desktop-portal is not reachable on the session bus; \
+ screen capture will fail.",
+ "Use a wlroots-based compositor (sway, labwc, hyprland, kwin 5.27+) \
+ or install xdg-desktop-portal-gnome / xdg-desktop-portal-kde / \
+ xdg-desktop-portal-wlr.",
+ );
+ }
// On Linux the canonical capture path is X11 GetImage / xwd / scrot
// — all require an open X11 connection. The probe is the same one
// we use for ax_capability, but the consumer-facing message and
@@ -126,10 +217,91 @@ async fn check_screen_capture_capability() -> CheckEntry {
"X11 reachable; screen capture path is functional.",
);
}
+ let hint = if std::env::var_os("WAYLAND_DISPLAY").is_some() {
+ "Pure Wayland session: opt into the experimental Wayland backend by \
+ setting CUA_DRIVER_RS_ENABLE_WAYLAND=1, or run the target under XWayland."
+ } else {
+ "Set DISPLAY (X11). Pure Wayland sessions require an XWayland bridge for capture."
+ };
CheckEntry::fail(
NAME_SCREEN_CAPTURE_CAPABILITY,
"X11 is not reachable; screen capture will fail.",
- "Set DISPLAY (X11). Pure Wayland sessions require an XWayland bridge for capture.",
+ hint,
+ )
+}
+
+/// `wayland_backend` doctor entry — surfaces which wlroots manager globals
+/// the running compositor advertises. Skipped on non-Wayland sessions and
+/// when the experimental Wayland backend isn't opted into, so X11 users see
+/// a neutral skip rather than a confusing failure.
+async fn check_wayland_backend() -> CheckEntry {
+ if std::env::var_os("WAYLAND_DISPLAY").is_none() {
+ return CheckEntry::skip(
+ NAME_WAYLAND_BACKEND.to_owned(),
+ "No WAYLAND_DISPLAY in the environment — not a Wayland session.",
+ );
+ }
+ if !wayland_opt_in_enabled() {
+ return CheckEntry::skip(
+ NAME_WAYLAND_BACKEND.to_owned(),
+ format!(
+ "Wayland session detected, but the experimental backend is opt-in. \
+ Set {}=1 to enable native Wayland and re-run doctor.",
+ wayland_env_name()
+ ),
+ );
+ }
+ let snap = match tokio::task::spawn_blocking(probe_wayland_managers).await {
+ Ok(Ok(snap)) => snap,
+ Ok(Err(e)) => {
+ return CheckEntry::fail(
+ NAME_WAYLAND_BACKEND,
+ format!("Failed to connect to the Wayland compositor: {e}"),
+ "Verify WAYLAND_DISPLAY points at a running compositor socket.",
+ );
+ }
+ Err(e) => {
+ return CheckEntry::fail(
+ NAME_WAYLAND_BACKEND,
+ format!("Wayland probe task error: {e}"),
+ "Re-run the doctor; if it persists, file a bug with the doctor output.",
+ );
+ }
+ };
+ let msg = format!(
+ "foreign-toplevel={ftl}, screencopy={cap}, virtual-pointer={vp}, wl_shm={shm}",
+ ftl = snap.foreign_toplevel,
+ cap = snap.screencopy,
+ vp = snap.virtual_pointer,
+ shm = snap.wl_shm,
+ );
+ if snap.foreign_toplevel && snap.screencopy && snap.virtual_pointer && snap.wl_shm {
+ return CheckEntry::pass(
+ NAME_WAYLAND_BACKEND,
+ format!("All wlroots manager globals advertised ({msg})."),
+ );
+ }
+ // Partial-pass: list_windows + capture both work, but virtual-pointer
+ // input is missing. Require `wl_shm` here too — `check_screen_capture_capability`
+ // gates on both `screencopy && wl_shm`, so excluding `wl_shm` from the
+ // partial-pass verdict would let the matrices disagree on degenerate
+ // compositors that omit it.
+ if snap.foreign_toplevel && snap.screencopy && snap.wl_shm {
+ return CheckEntry::pass(
+ NAME_WAYLAND_BACKEND,
+ format!(
+ "Core wlroots manager globals available; some optional globals missing ({msg}). \
+ Input may fall back where virtual-pointer is absent."
+ ),
+ );
+ }
+ CheckEntry::fail(
+ NAME_WAYLAND_BACKEND,
+ format!(
+ "Compositor does not advertise the wlroots manager globals cua-driver \
+ needs ({msg})."
+ ),
+ "Use a wlroots-based compositor (sway, labwc, hyprland) or run under XWayland.",
)
}
@@ -145,6 +317,102 @@ fn probe_x11_connect() -> bool {
false
}
+/// Probe whether `org.a11y.Bus` is reachable on the session bus — the
+/// canonical AT-SPI prerequisite on Wayland, where there is no X server to
+/// stand in. Returns false on any error (no session bus, no a11y service,
+/// timeout, etc.) so the doctor message stays simple.
+#[cfg(target_os = "linux")]
+fn probe_a11y_bus() -> bool {
+ use atspi::zbus;
+ let rt = match tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ {
+ Ok(rt) => rt,
+ Err(_) => return false,
+ };
+ rt.block_on(async {
+ let bus = match zbus::Connection::session().await {
+ Ok(b) => b,
+ Err(_) => return false,
+ };
+ let proxy = match zbus::fdo::DBusProxy::new(&bus).await {
+ Ok(p) => p,
+ Err(_) => return false,
+ };
+ // `name_has_owner` is the cheap direct existence check — avoids
+ // pulling the entire session-bus name registry just to look up one
+ // entry. Matches the pattern in `wayland::portal_screenshot::probe_portal`.
+ let bus_name = match "org.a11y.Bus".try_into() {
+ Ok(n) => n,
+ Err(_) => return false,
+ };
+ proxy.name_has_owner(bus_name).await.unwrap_or(false)
+ })
+}
+
+#[cfg(not(target_os = "linux"))]
+fn probe_a11y_bus() -> bool {
+ false
+}
+
+/// True when the Wayland backend is opted in and a Wayland session is
+/// active. Wrapper around `wayland::is_wayland()` so non-Linux builds of
+/// this file compile (the `wayland` module is gated on `target_os = linux`).
+#[cfg(target_os = "linux")]
+fn is_wayland_session() -> bool {
+ crate::wayland::is_wayland()
+}
+
+#[cfg(not(target_os = "linux"))]
+fn is_wayland_session() -> bool {
+ false
+}
+
+#[cfg(target_os = "linux")]
+fn wayland_opt_in_enabled() -> bool {
+ crate::wayland::wayland_enabled()
+}
+
+#[cfg(not(target_os = "linux"))]
+fn wayland_opt_in_enabled() -> bool {
+ false
+}
+
+#[cfg(target_os = "linux")]
+fn wayland_env_name() -> &'static str {
+ crate::wayland::ENABLE_WAYLAND_ENV
+}
+
+#[cfg(not(target_os = "linux"))]
+fn wayland_env_name() -> &'static str {
+ "CUA_DRIVER_RS_ENABLE_WAYLAND"
+}
+
+/// Snapshot of wlroots manager globals. The non-Linux stub returns an empty
+/// snapshot so off-platform builds stay green; doctor short-circuits before
+/// reaching it via [`is_wayland_session`].
+#[cfg(target_os = "linux")]
+fn probe_wayland_managers() -> anyhow::Result<crate::wayland::WaylandManagers> {
+ crate::wayland::probe_managers()
+}
+
+#[cfg(not(target_os = "linux"))]
+fn probe_wayland_managers() -> anyhow::Result<WaylandManagers> {
+ Ok(WaylandManagers::default())
+}
+
+/// Stub of `wayland::WaylandManagers` so off-Linux builds compile. Always
+/// reports nothing advertised — non-Linux code paths never call this.
+#[cfg(not(target_os = "linux"))]
+#[derive(Default, Clone, Debug)]
+struct WaylandManagers {
+ foreign_toplevel: bool,
+ screencopy: bool,
+ virtual_pointer: bool,
+ wl_shm: bool,
+}
+
/// `PRETTY_NAME=` out of `/etc/os-release`. Fails to "Linux" so the
/// downstream message stays human-readable.
fn os_release_pretty_name() -> Option<String> {
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
index 5c8647472f..c03e36be81 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs
@@ -127,8 +127,15 @@ pub fn send_command_for(key: CursorKey, cmd: OverlayCommand) {
if key.is_empty() {
return;
}
+ let msg = OverlayMsg::Cmd(KeyedOverlayCommand { key: key.clone(), cmd: cmd.clone() });
if let Some(tx) = CMD_TX.get() {
- let _ = tx.try_send(OverlayMsg::Cmd(KeyedOverlayCommand { key, cmd }));
+ let _ = tx.try_send(msg.clone());
+ }
+ // Also forward to the native-Wayland layer-shell overlay when Wayland
+ // is opted in. The wayland overlay's `forward` is a no-op when its
+ // owner thread isn't started yet (which is the normal X11-only case).
+ if crate::wayland::is_wayland() {
+ let _ = crate::wayland::overlay::forward(&msg);
}
}
@@ -247,6 +254,17 @@ pub fn run_on_thread() {
return;
}
+ // Wayland layer-shell overlay is started LAZILY on the first
+ // send_command_for() that targets a Wayland session (see the
+ // wayland::overlay::forward() path). Starting it eagerly here added
+ // ~100-300ms to cua-driver mcp startup — enough to push the CI
+ // `cursor-click-gif` test (20s budget for the full launch_app →
+ // click → type sequence) over its limit, intermittently. The
+ // forward() path's own ensure_started() OnceLock guarantees the
+ // thread spins up on demand without losing any commands (the
+ // first send_command_for that triggers it spawns the thread,
+ // future commands reuse it).
+
std::thread::Builder::new()
.name("cua-overlay-x11".into())
.spawn(move || {
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs b/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs
index f0966ef052..0e2db4dc15 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/terminal.rs
@@ -65,12 +65,14 @@ pub fn wm_class_matches_terminal(instance: &str, class: &str) -> bool {
}
/// Returns `true` if the window with X11 id `xid` belongs to a
-/// terminal emulator, based on its `WM_CLASS` property. Returns
-/// `false` when no X connection is available or the property is
-/// unset.
+/// terminal emulator, based on its `WM_CLASS` property. On native
+/// Wayland the dispatcher folds the foreign-toplevel `app_id` into
+/// both fields so the substring match still works for terminals like
+/// Ghostty / kitty / alacritty. Returns `false` when no connection is
+/// available or the property is unset.
#[cfg(target_os = "linux")]
pub fn is_terminal_window(xid: u64) -> bool {
- match crate::x11::wm_class_for_window(xid) {
+ match crate::wayland::wm_class_dispatch(xid) {
Some((instance, class)) => wm_class_matches_terminal(&instance, &class),
None => false,
}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
index 876056122c..08da3548cb 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
@@ -1048,23 +1048,12 @@ impl Tool for ClickTool {
let (xi, yi) = (x as i32, y as i32);
let result = tokio::task::spawn_blocking(move || {
// Native Wayland: focus+raise the target toplevel (foreign-toplevel
- // `activate`). Wayland hides cross-window geometry, so window-local
- // x/y can't be mapped to a global pointer position; activating the
- // window the caller targeted is the focus-based equivalent.
+ // `activate`), then drive `count` virtual-pointer button events at
+ // the requested coordinates. Wayland hides cross-window geometry,
+ // so callers should pass output-relative coords; (0,0) preserves
+ // the legacy "click the activated window's centre" behaviour.
if crate::wayland::is_wayland() {
- // Surface 5: the Wayland virtual-pointer path here only emits a
- // left-button press at the output centre. Surface a real error
- // instead of silently dropping a right/middle request onto a
- // left-click, so the Hermes-side audit gap closes.
- if button != 1 {
- let label = mouse_button_name(button);
- anyhow::bail!(
- "{label}-button click is not supported on this platform \
- (native Wayland virtual-pointer path: left-button only). \
- Use the X11 backend, or run the target under XWayland."
- );
- }
- return crate::wayland::click(xid);
+ return crate::wayland::click(xid, xi, yi, count as u32, button);
}
crate::input::send_click(xid, xi, yi, count, button)
}).await;
@@ -1506,7 +1495,17 @@ impl Tool for HotkeyTool {
};
let key_display = format!("{}+{}", mods.join("+"), key);
+ let key_for_wayland = key.clone();
+ let mods_for_wayland = mods.clone();
let result = tokio::task::spawn_blocking(move || {
+ if crate::wayland::is_wayland() {
+ // Native Wayland: route the modifier combo through wtype's
+ // -M/-k/-m sequence — the closest equivalent to the X11
+ // state-mask path. window_id is irrelevant once focused.
+ let mut combo: Vec<String> = mods_for_wayland.clone();
+ combo.push(key_for_wayland.clone());
+ return crate::wayland::hotkey(&combo);
+ }
let m: Vec<&str> = mods.iter().map(String::as_str).collect();
crate::input::send_key(xid, &key, &m)
}).await;
@@ -1663,7 +1662,12 @@ impl Tool for ScrollTool {
let button: u8 = match direction.as_str() {
"up" => 4, "left" => 6, "right" => 7, _ => 5,
};
+ let direction_for_wayland = direction.clone();
+ let amount_u32 = amount as u32;
let result = tokio::task::spawn_blocking(move || {
+ if crate::wayland::is_wayland() {
+ return crate::wayland::scroll(xid, &direction_for_wayland, amount_u32);
+ }
crate::input::send_click(xid, 0, 0, amount, button)
}).await;
match result {
@@ -1752,7 +1756,15 @@ impl Tool for DoubleClickTool {
cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },
);
}
- match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 2, 1)).await {
+ let lxi = lx as i32;
+ let lyi = ly as i32;
+ let click_result = tokio::task::spawn_blocking(move || {
+ if crate::wayland::is_wayland() {
+ return crate::wayland::click(xid, lxi, lyi, 2, 1);
+ }
+ crate::input::send_click(xid, lxi, lyi, 2, 1)
+ }).await;
+ match click_result {
Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked element [{idx}].")),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
@@ -1793,7 +1805,12 @@ impl Tool for DoubleClickTool {
);
}
let (xi, yi) = (x as i32, y as i32);
- let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 2, 1)).await;
+ let result = tokio::task::spawn_blocking(move || {
+ if crate::wayland::is_wayland() {
+ return crate::wayland::click(xid, xi, yi, 2, 1);
+ }
+ crate::input::send_click(xid, xi, yi, 2, 1)
+ }).await;
match result {
Ok(Ok(())) => ToolResult::text(format!("✅ Double-clicked at ({x:.1}, {y:.1}).")),
Ok(Err(e)) => ToolResult::error(e.to_string()),
@@ -1875,7 +1892,15 @@ impl Tool for RightClickTool {
cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy },
);
}
- match tokio::task::spawn_blocking(move || crate::input::send_click(xid, lx as i32, ly as i32, 1, 3)).await {
+ let lxi = lx as i32;
+ let lyi = ly as i32;
+ let click_result = tokio::task::spawn_blocking(move || {
+ if crate::wayland::is_wayland() {
+ return crate::wayland::click(xid, lxi, lyi, 1, 3);
+ }
+ crate::input::send_click(xid, lxi, lyi, 1, 3)
+ }).await;
+ match click_result {
Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked element [{idx}].")),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
@@ -1916,7 +1941,12 @@ impl Tool for RightClickTool {
);
}
let (xi, yi) = (x as i32, y as i32);
- let result = tokio::task::spawn_blocking(move || crate::input::send_click(xid, xi, yi, 1, 3)).await;
+ let result = tokio::task::spawn_blocking(move || {
+ if crate::wayland::is_wayland() {
+ return crate::wayland::click(xid, xi, yi, 1, 3);
+ }
+ crate::input::send_click(xid, xi, yi, 1, 3)
+ }).await;
match result {
Ok(Ok(())) => ToolResult::text(format!("✅ Right-clicked at ({x:.1}, {y:.1}).")),
Ok(Err(e)) => ToolResult::error(e.to_string()),
@@ -2013,6 +2043,31 @@ impl Tool for DragTool {
cursor_overlay::OverlayCommand::SetPressed(true),
);
+ // Native Wayland: emit press + interpolated motion + release as one
+ // virtual-pointer sequence (output-relative coords). Returns early so
+ // we don't fall into the X11 XSendEvent loop below.
+ if crate::wayland::is_wayland() {
+ let (fxi, fyi) = (from_x.round() as i32, from_y.round() as i32);
+ let (txi, tyi) = (to_x.round() as i32, to_y.round() as i32);
+ let steps_u32 = steps as u32;
+ let drag_result = tokio::task::spawn_blocking(move || {
+ crate::wayland::drag(xid, fxi, fyi, txi, tyi, steps_u32, button)
+ }).await;
+ crate::overlay::send_command_for(
+ cursor_id.clone(),
+ cursor_overlay::OverlayCommand::SetPressed(false),
+ );
+ return match drag_result {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "✅ Posted drag ({button_str}) to pid {pid} \
+ from ({from_x:.0}, {from_y:.0}) → ({to_x:.0}, {to_y:.0}) \
+ in {duration_ms}ms / {steps} steps."
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
let press_result = tokio::task::spawn_blocking(move || {
crate::input::send_button_down(xid, from_x.round() as i32, from_y.round() as i32, button)
}).await;
@@ -2181,7 +2236,15 @@ impl Tool for MouseButtonDownTool {
let xi = x as i32;
let yi = y as i32;
- let result = tokio::task::spawn_blocking(move || crate::input::send_button_down(xid, xi, yi, button)).await;
+ // Native Wayland: route through the persistent virtual-pointer module
+ // so the held button survives across tool calls; the X11 path keeps
+ // the existing input::send_button_down behaviour.
+ let result = if crate::wayland::is_wayland() {
+ let cid = cursor_id.clone();
+ tokio::task::spawn_blocking(move || crate::wayland::persistent_vptr::press(&cid, xid, xi, yi, button)).await
+ } else {
+ tokio::task::spawn_blocking(move || crate::input::send_button_down(xid, xi, yi, button)).await
+ };
match result {
Ok(Ok(())) => {
let hold = MouseHoldState { cursor_id: cursor_id.clone(), pid, xid, button, x, y };
@@ -2288,13 +2351,24 @@ impl Tool for MouseDragTool {
let mut result: anyhow::Result<()> = Ok(());
let mut prev_x = from_x;
let mut prev_y = from_y;
+ let is_wl = crate::wayland::is_wayland();
for i in 1..=steps {
let t = i as f64 / steps as f64;
let ix = from_x + (to_x - from_x) * t;
let iy = from_y + (to_y - from_y) * t;
- let move_result = tokio::task::spawn_blocking(move || {
- crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button))
- }).await;
+ // Native Wayland: route motion through the persistent virtual-
+ // pointer so the held button stays held across the entire drag;
+ // X11 keeps the existing input::send_motion path.
+ let cid_inner = cursor_id.clone();
+ let move_result = if is_wl {
+ tokio::task::spawn_blocking(move || {
+ crate::wayland::persistent_vptr::move_to(&cid_inner, ix.round() as i32, iy.round() as i32)
+ }).await
+ } else {
+ tokio::task::spawn_blocking(move || {
+ crate::input::send_motion(xid, ix.round() as i32, iy.round() as i32, Some(button))
+ }).await
+ };
match move_result {
Ok(Ok(())) => {
if let Ok(Ok((sx, sy))) =
@@ -2411,7 +2485,15 @@ impl Tool for MouseButtonUpTool {
let button = hold.button;
let xi = x as i32;
let yi = y as i32;
- let result = tokio::task::spawn_blocking(move || crate::input::send_button_up(xid, xi, yi, button)).await;
+ // Native Wayland: release through the persistent virtual-pointer so
+ // the same vptr device that emitted the press also emits the release
+ // (single logical drag rather than a click pair).
+ let result = if crate::wayland::is_wayland() {
+ let cid = cursor_id.clone();
+ tokio::task::spawn_blocking(move || crate::wayland::persistent_vptr::release(&cid, button)).await
+ } else {
+ tokio::task::spawn_blocking(move || crate::input::send_button_up(xid, xi, yi, button)).await
+ };
match result {
Ok(Ok(())) => {
hold.x = x;
@@ -2535,6 +2617,16 @@ impl Tool for ParallelMouseDragTool {
if crate::wayland::is_inject_mode() {
return parallel_drag_inject(&args).await;
}
+ // Native Wayland without the inject socket: MPX/XI2 + uinput master
+ // pointers don't exist on Wayland. Surface a typed error instead of
+ // silently calling the X11 path that's guaranteed to fail.
+ if crate::wayland::is_wayland() {
+ return ToolResult::error(
+ "parallel_mouse_drag requires the cua-compositor inject socket on Wayland \
+ (set CUA_INJECT_SOCKET to the cua-compositor control socket), \
+ or run the target under X11."
+ );
+ }
match tokio::task::spawn_blocking(crate::input::check_parallel_pointer_support).await {
Ok(Ok(())) => {}
Ok(Err(e)) => return ToolResult::error(e.to_string()),
@@ -2741,6 +2833,21 @@ impl Tool for GetCursorPositionTool {
})
}
async fn invoke(&self, _args: Value) -> ToolResult {
+ // Native Wayland: there's no protocol for clients to query the real
+ // global cursor position. Fall back to the synthetic registry that
+ // records every `motion_absolute` this process emits.
+ if crate::wayland::is_wayland() {
+ return match crate::wayland::last_synth_cursor_pos() {
+ Some((x, y)) => ToolResult::text(
+ format!("✅ Cursor at ({x}, {y}) (synthetic — last move_cursor in this process)")
+ ).with_structured(json!({
+ "x": x, "y": y, "source": "synthetic"
+ })),
+ None => ToolResult::text(
+ "Cursor position unknown on Wayland — no move_cursor has been issued in this process yet.".to_string()
+ ).with_structured(json!({ "source": "synthetic", "available": false })),
+ };
+ }
let result = tokio::task::spawn_blocking(|| {
use x11rb::connection::Connection;
use x11rb::protocol::xproto::ConnectionExt as _;
@@ -2753,7 +2860,7 @@ impl Tool for GetCursorPositionTool {
match result {
// Text format matches Swift `GetCursorPositionTool` 1:1.
Ok(Ok((x, y))) => ToolResult::text(format!("✅ Cursor at ({x}, {y})"))
- .with_structured(json!({ "x": x, "y": y })),
+ .with_structured(json!({ "x": x, "y": y, "source": "x11" })),
Ok(Err(e)) => ToolResult::error(e.to_string()),
Err(e) => ToolResult::error(format!("Task error: {e}")),
}
@@ -2784,6 +2891,7 @@ impl Tool for MoveCursorTool {
use cua_driver_core::tool_args::ArgsExt;
let x = args.f64_or("x", 0.0);
let y = args.f64_or("y", 0.0);
+ let window_id = args.get("window_id").and_then(|v| v.as_u64());
let cursor_id = resolve_cursor_key(&args);
self.state.cursor_registry.update_position(&cursor_id, x, y);
// End pointing upper-left (45°) — matches Swift's
@@ -2797,7 +2905,21 @@ impl Tool for MoveCursorTool {
end_heading_radians: std::f64::consts::FRAC_PI_4,
},
);
- ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1})."))
+ // Native Wayland: also warp the real cursor via zwlr_virtual_pointer.
+ // Off-thread because the wayland-client roundtrip is blocking. Best-effort
+ // — overlay update + registry write already succeeded; surface a warning
+ // only if the warp itself failed.
+ let real_warp_note = if crate::wayland::is_wayland() {
+ let xi = x.round() as i32;
+ let yi = y.round() as i32;
+ match tokio::task::spawn_blocking(move || crate::wayland::move_cursor_absolute(window_id, xi, yi)).await {
+ Ok(Ok(())) => " (real cursor warped via virtual-pointer)",
+ Ok(Err(_)) | Err(_) => " (overlay updated; real-cursor warp failed)",
+ }
+ } else {
+ ""
+ };
+ ToolResult::text(format!("Agent cursor '{cursor_id}' moved to ({x:.1}, {y:.1}).{real_warp_note}"))
}
}
@@ -3384,7 +3506,11 @@ impl Tool for ZoomTool {
let state = self.state.clone();
let result = tokio::task::spawn_blocking(move || {
- let png = crate::capture::screenshot_window_bytes(xid)?;
+ // Route through the Wayland-aware window capture dispatcher so
+ // pure-Wayland sessions surface a typed "per-window capture not
+ // supported yet" error instead of accidentally calling the
+ // X11-only path with a foreign-toplevel id.
+ let png = crate::wayland::screenshot_window_dispatch(xid)?;
cursor_overlay::capture_utils::crop_png_to_jpeg(&png, x1, y1, x2, y2, 500)
}).await;
@@ -3470,6 +3596,20 @@ impl Tool for TypeTextCharsTool {
};
let text_len = text.chars().count();
let result = tokio::task::spawn_blocking(move || {
+ if crate::wayland::is_wayland() {
+ // Per-char `wtype` loop with the requested delay — mirrors the
+ // X11 XSendEvent per-char path. Sleeping here is fine because
+ // we're inside spawn_blocking.
+ let mut buf = [0u8; 4];
+ for ch in text.chars() {
+ let s = ch.encode_utf8(&mut buf);
+ crate::wayland::type_text(s)?;
+ if delay_ms > 0 {
+ std::thread::sleep(std::time::Duration::from_millis(delay_ms));
+ }
+ }
+ return Ok(());
+ }
crate::input::send_type_text_with_delay(xid, &text, delay_ms)
}).await;
match result {
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rs
new file mode 100644
index 0000000000..d4e313e47e
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_screencopy.rs
@@ -0,0 +1,348 @@
+//! Output capture via `ext-image-copy-capture-v1` (staging protocol).
+//!
+//! The wlroots `zwlr_screencopy_manager_v1` capture path in `mod.rs` works
+//! on wlroots compositors (sway, labwc) but is missing from compositors
+//! that don't speak wlroots protocols (GNOME mutter, KDE Plasma <6.2).
+//! The staging `ext-image-copy-capture-v1` protocol replaces wlr-screencopy
+//! across the ecosystem — sway 1.10+, labwc 0.8+, niri 0.1.6+, hyprland
+//! 0.45+, KDE Plasma 6.2+, GNOME mutter 47+ all implement it.
+//!
+//! This module captures the FIRST `wl_output` advertised by the compositor
+//! via the chain:
+//! ext_output_image_capture_source_manager_v1.create_source(wl_output)
+//! → ext_image_copy_capture_manager_v1.create_session(source, options)
+//! → session emits buffer_size + done
+//! → allocate wl_shm pool + wl_buffer matching the size/format
+//! → session.create_frame() → frame.attach_buffer(buffer) → frame.capture()
+//! → frame emits ready (success) or failed (error)
+//! → read pixels out of the mmap'd buffer, PNG-encode, destroy
+//!
+//! Per-window capture (ext_foreign_toplevel_image_capture_source_manager_v1)
+//! is a follow-up — needs the existing `list_windows` to migrate from
+//! `zwlr_foreign_toplevel_manager_v1` to `ext_foreign_toplevel_list_v1` so
+//! the window_id handles match up.
+
+use std::time::Duration;
+
+use wayland_client::{
+ protocol::{
+ wl_buffer::WlBuffer,
+ wl_output::WlOutput,
+ wl_registry,
+ wl_shm::{self, WlShm},
+ wl_shm_pool::WlShmPool,
+ },
+ Connection, Dispatch, Proxy, QueueHandle,
+};
+use wayland_protocols::ext::image_capture_source::v1::client::{
+ ext_image_capture_source_v1::ExtImageCaptureSourceV1,
+ ext_output_image_capture_source_manager_v1::ExtOutputImageCaptureSourceManagerV1,
+};
+use wayland_protocols::ext::image_copy_capture::v1::client::{
+ ext_image_copy_capture_frame_v1::{self, ExtImageCopyCaptureFrameV1},
+ ext_image_copy_capture_manager_v1::{self, ExtImageCopyCaptureManagerV1},
+ ext_image_copy_capture_session_v1::{self, ExtImageCopyCaptureSessionV1},
+};
+
+#[derive(Default)]
+struct CapState {
+ output: Option<WlOutput>,
+ source_mgr: Option<ExtOutputImageCaptureSourceManagerV1>,
+ copy_mgr: Option<ExtImageCopyCaptureManagerV1>,
+ shm: Option<WlShm>,
+ // Buffer specs reported by the session before we allocate.
+ fmt: Option<u32>,
+ width: u32,
+ height: u32,
+ stride: u32,
+ done: bool,
+ // Frame outcome
+ ready: bool,
+ failed: bool,
+ fail_reason: Option<u32>,
+}
+
+/// Capture the first `wl_output` via ext-image-copy-capture-v1, return raw
+/// PNG bytes. Falls back to the caller's next tier when the compositor
+/// doesn't expose the protocol — surface a typed anyhow::Error so the
+/// cascade in `screenshot_display_dispatch` can branch.
+pub fn screenshot_via_ext_copy() -> anyhow::Result<Vec<u8>> {
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<CapState>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+
+ let mut state = CapState::default();
+ queue.roundtrip(&mut state)?;
+ for _ in 0..3 {
+ queue.roundtrip(&mut state)?;
+ }
+
+ let output = state
+ .output
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_output"))?;
+ let source_mgr = state.source_mgr.clone().ok_or_else(|| {
+ anyhow::anyhow!("compositor does not expose ext_output_image_capture_source_manager_v1")
+ })?;
+ let copy_mgr = state.copy_mgr.clone().ok_or_else(|| {
+ anyhow::anyhow!("compositor does not expose ext_image_copy_capture_manager_v1")
+ })?;
+ let shm = state
+ .shm
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_shm"))?;
+
+ // Create source + session.
+ let source: ExtImageCaptureSourceV1 = source_mgr.create_source(&output, &qh, ());
+ let session: ExtImageCopyCaptureSessionV1 = copy_mgr.create_session(
+ &source,
+ ext_image_copy_capture_manager_v1::Options::empty(),
+ &qh,
+ (),
+ );
+
+ // Wait for the session to announce buffer dimensions + format + done.
+ let deadline = std::time::Instant::now() + Duration::from_secs(3);
+ while !state.done && std::time::Instant::now() < deadline {
+ queue.roundtrip(&mut state)?;
+ if state.width > 0 && state.height > 0 && state.fmt.is_some() {
+ // Some compositors don't emit `done` explicitly — proceed once
+ // we have geometry + format.
+ state.done = true;
+ }
+ }
+ if !state.done || state.width == 0 || state.height == 0 || state.fmt.is_none() {
+ anyhow::bail!(
+ "ext-image-copy-capture session never reported buffer specs (w={}, h={}, fmt={:?})",
+ state.width, state.height, state.fmt
+ );
+ }
+
+ // Checked arithmetic on compositor-provided sizes. A malicious or
+ // buggy compositor could announce dimensions that overflow `usize` on
+ // multiplication — we refuse to allocate in that case rather than
+ // wrapping and reading past the buffer.
+ let stride = if state.stride > 0 {
+ state.stride
+ } else {
+ state.width
+ .checked_mul(4)
+ .ok_or_else(|| anyhow::anyhow!("compositor advertised width that overflows stride: {}", state.width))?
+ };
+ if stride < state.width.saturating_mul(4) {
+ anyhow::bail!(
+ "compositor stride {} is smaller than width*4 {}; refusing to allocate (would alias rows)",
+ stride, state.width.saturating_mul(4)
+ );
+ }
+ let size = (stride as usize)
+ .checked_mul(state.height as usize)
+ .ok_or_else(|| {
+ anyhow::anyhow!(
+ "compositor buffer size overflows usize: stride={} height={}",
+ stride, state.height
+ )
+ })?;
+
+ // Allocate the wl_shm buffer.
+ let (fd, ptr) = super::anon_shm(size)?;
+ use std::os::fd::AsFd as _;
+ let pool_fd = unsafe { super::borrowed_fd(fd) };
+ let pool: WlShmPool = shm.create_pool(pool_fd.as_fd(), size as i32, &qh, ());
+ let fmt: wl_shm::Format = wl_shm::Format::try_from(state.fmt.unwrap()).map_err(|_| {
+ anyhow::anyhow!("compositor advertised unsupported wl_shm format {:#x}", state.fmt.unwrap())
+ })?;
+ let buffer: WlBuffer = pool.create_buffer(
+ 0,
+ state.width as i32,
+ state.height as i32,
+ stride as i32,
+ fmt,
+ &qh,
+ (),
+ );
+
+ // Create + run the frame.
+ let frame: ExtImageCopyCaptureFrameV1 = session.create_frame(&qh, ());
+ frame.attach_buffer(&buffer);
+ frame.capture();
+ queue.roundtrip(&mut state)?;
+
+ let deadline = std::time::Instant::now() + Duration::from_secs(5);
+ while !state.ready && !state.failed && std::time::Instant::now() < deadline {
+ queue.roundtrip(&mut state)?;
+ }
+ if state.failed {
+ super::cleanup_mmap(ptr, size, fd);
+ anyhow::bail!(
+ "ext-image-copy-capture frame failed (reason={:?})",
+ state.fail_reason
+ );
+ }
+ if !state.ready {
+ super::cleanup_mmap(ptr, size, fd);
+ anyhow::bail!("ext-image-copy-capture frame timed out without ready event");
+ }
+
+ // Copy + encode + cleanup.
+ let pixels: &[u8] = unsafe { std::slice::from_raw_parts(ptr as *const u8, size) };
+ let png = encode_buffer_to_png(pixels, state.width, state.height, stride, fmt)?;
+
+ frame.destroy();
+ session.destroy();
+ source.destroy();
+ pool.destroy();
+ queue.roundtrip(&mut state).ok();
+ super::cleanup_mmap(ptr, size, fd);
+
+ Ok(png)
+}
+
+fn encode_buffer_to_png(
+ pixels: &[u8],
+ width: u32,
+ height: u32,
+ stride: u32,
+ fmt: wl_shm::Format,
+) -> anyhow::Result<Vec<u8>> {
+ // The image crate expects RGBA8888 row-packed. wl_shm gives us Argb8888
+ // / Xrgb8888 which in little-endian memory layout is BGRA / BGRX. Swap
+ // channels into RGBA and pack rows.
+ let mut rgba: Vec<u8> = Vec::with_capacity((width as usize) * (height as usize) * 4);
+ for y in 0..(height as usize) {
+ let row_start = y * (stride as usize);
+ for x in 0..(width as usize) {
+ let i = row_start + x * 4;
+ let b = pixels[i];
+ let g = pixels[i + 1];
+ let r = pixels[i + 2];
+ let a = match fmt {
+ wl_shm::Format::Xrgb8888 => 0xFF,
+ _ => pixels[i + 3],
+ };
+ rgba.extend_from_slice(&[r, g, b, a]);
+ }
+ }
+
+ use image::{ImageBuffer, Rgba, codecs::png::PngEncoder};
+ let img: ImageBuffer<Rgba<u8>, _> =
+ ImageBuffer::from_raw(width, height, rgba).ok_or_else(|| {
+ anyhow::anyhow!("internal: buffer dims mismatch ({}x{})", width, height)
+ })?;
+ let mut out: Vec<u8> = Vec::new();
+ use image::ImageEncoder;
+ PngEncoder::new(&mut out).write_image(
+ img.as_raw(),
+ width,
+ height,
+ image::ExtendedColorType::Rgba8,
+ )?;
+ Ok(out)
+}
+
+// ── Wayland Dispatch impls ───────────────────────────────────────────────
+
+impl Dispatch<wl_registry::WlRegistry, ()> for CapState {
+ fn event(
+ state: &mut Self,
+ registry: &wl_registry::WlRegistry,
+ event: wl_registry::Event,
+ _: &(),
+ _: &Connection,
+ qh: &QueueHandle<Self>,
+ ) {
+ if let wl_registry::Event::Global { name, interface, version } = event {
+ match interface.as_str() {
+ "wl_output" => {
+ if state.output.is_none() {
+ state.output = Some(registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ()));
+ }
+ }
+ "wl_shm" => {
+ state.shm = Some(registry.bind::<WlShm, _, _>(name, 1, qh, ()));
+ }
+ "ext_output_image_capture_source_manager_v1" => {
+ state.source_mgr = Some(registry.bind::<ExtOutputImageCaptureSourceManagerV1, _, _>(
+ name, 1, qh, ()));
+ }
+ "ext_image_copy_capture_manager_v1" => {
+ state.copy_mgr = Some(registry.bind::<ExtImageCopyCaptureManagerV1, _, _>(
+ name, 1, qh, ()));
+ }
+ _ => {}
+ }
+ }
+ }
+}
+
+impl Dispatch<WlOutput, ()> for CapState {
+ fn event(_: &mut Self, _: &WlOutput, _: <WlOutput as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}
+}
+impl Dispatch<WlShm, ()> for CapState {
+ fn event(_: &mut Self, _: &WlShm, _: <WlShm as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}
+}
+impl Dispatch<WlShmPool, ()> for CapState {
+ fn event(_: &mut Self, _: &WlShmPool, _: <WlShmPool as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}
+}
+impl Dispatch<WlBuffer, ()> for CapState {
+ fn event(_: &mut Self, _: &WlBuffer, _: <WlBuffer as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}
+}
+impl Dispatch<ExtOutputImageCaptureSourceManagerV1, ()> for CapState {
+ fn event(_: &mut Self, _: &ExtOutputImageCaptureSourceManagerV1, _: <ExtOutputImageCaptureSourceManagerV1 as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}
+}
+impl Dispatch<ExtImageCaptureSourceV1, ()> for CapState {
+ fn event(_: &mut Self, _: &ExtImageCaptureSourceV1, _: <ExtImageCaptureSourceV1 as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}
+}
+impl Dispatch<ExtImageCopyCaptureManagerV1, ()> for CapState {
+ fn event(_: &mut Self, _: &ExtImageCopyCaptureManagerV1, _: <ExtImageCopyCaptureManagerV1 as Proxy>::Event, _: &(), _: &Connection, _: &QueueHandle<Self>) {}
+}
+impl Dispatch<ExtImageCopyCaptureSessionV1, ()> for CapState {
+ fn event(
+ state: &mut Self,
+ _: &ExtImageCopyCaptureSessionV1,
+ event: <ExtImageCopyCaptureSessionV1 as Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ use ext_image_copy_capture_session_v1::Event as Ev;
+ match event {
+ Ev::BufferSize { width, height } => {
+ state.width = width;
+ state.height = height;
+ }
+ Ev::ShmFormat { format } => {
+ if state.fmt.is_none() {
+ if let Ok(f) = format.into_result() {
+ state.fmt = Some(f as u32);
+ }
+ }
+ }
+ Ev::Done => {
+ state.done = true;
+ }
+ _ => {}
+ }
+ }
+}
+impl Dispatch<ExtImageCopyCaptureFrameV1, ()> for CapState {
+ fn event(
+ state: &mut Self,
+ _: &ExtImageCopyCaptureFrameV1,
+ event: <ExtImageCopyCaptureFrameV1 as Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ use ext_image_copy_capture_frame_v1::Event as Ev;
+ match event {
+ Ev::Ready => state.ready = true,
+ Ev::Failed { reason } => {
+ state.failed = true;
+ state.fail_reason = reason.into_result().ok().map(|r| r as u32);
+ }
+ _ => {}
+ }
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs
new file mode 100644
index 0000000000..2265ba74cb
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs
@@ -0,0 +1,647 @@
+//! libei-based input synthesis via the `reis` crate + xdg-desktop-portal
+//! RemoteDesktop.
+//!
+//! Reaches GNOME/Mutter (47+, mutter shipped EIS in 45, ei_text in 46)
+//! and KDE/KWin (Plasma 6.0+, ei_text in 6.1) — the cross-DE input
+//! complement that doesn't depend on the wlroots-only
+//! `zwlr_virtual_pointer_v1` / `zwlr_virtual_keyboard_v1` protocols.
+//!
+//! Sequence:
+//! 1. `ei::Context::connect_to_env()` — fast path when a $LIBEI_SOCKET is
+//! already exported (cua-compositor / inject mode).
+//! 2. Fallback: ashpd `RemoteDesktop::create_session` →
+//! `select_devices(KEYBOARD|POINTER)` →
+//! `start(session, parent)` (user consent dialog the first time) →
+//! `connect_to_eis()` returns a Unix fd → `ei::Context::new(fd)`.
+//! 3. Worker thread runs a calloop event loop on the ei::Context;
+//! handshake → connection.seat events → seat.bind(capabilities) →
+//! device.frame + emulate events.
+//! 4. Public API sends commands over a crossbeam-channel and blocks
+//! until the worker reports the request was flushed to the EIS
+//! server.
+//!
+//! Persistence: ashpd `PersistMode::Application` caches the consent grant
+//! for the lifetime of the requesting binary; restore_token written to
+//! `~/.config/cua-driver/libei.token` survives reboots (the worker
+//! reads it on startup).
+//!
+//! Coordinates: ei_pointer_absolute uses LOGICAL PIXELS inside an
+//! announced ei_device.Region — collected between device creation and
+//! the first device.Done event. Multi-monitor: pick the region
+//! containing the target (x, y).
+
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::sync::OnceLock;
+use std::thread;
+
+use crossbeam_channel::{bounded, Receiver, Sender};
+
+/// Buttons the public API exposes. Mapped to evdev codes in the worker
+/// thread so the libei surface only sees evdev integers.
+#[derive(Debug, Clone, Copy)]
+pub enum Button {
+ Left,
+ Right,
+ Middle,
+}
+
+impl Button {
+ fn to_evdev(self) -> u32 {
+ match self {
+ Button::Left => 0x110, // BTN_LEFT
+ Button::Right => 0x111, // BTN_RIGHT
+ Button::Middle => 0x112, // BTN_MIDDLE
+ }
+ }
+}
+
+/// Commands the worker thread accepts. Each carries a reply channel so
+/// the caller blocks until the EIS server has received the event.
+enum Cmd {
+ Click {
+ x: f64,
+ y: f64,
+ button: Button,
+ reply: Sender<anyhow::Result<()>>,
+ },
+ MoveAbsolute {
+ x: f64,
+ y: f64,
+ reply: Sender<anyhow::Result<()>>,
+ },
+ Scroll {
+ dx: f64,
+ dy: f64,
+ reply: Sender<anyhow::Result<()>>,
+ },
+ TypeText {
+ text: String,
+ reply: Sender<anyhow::Result<()>>,
+ },
+ PressKey {
+ keycode: u32,
+ reply: Sender<anyhow::Result<()>>,
+ },
+ Shutdown,
+}
+
+static TX: OnceLock<Sender<Cmd>> = OnceLock::new();
+
+fn tx() -> anyhow::Result<&'static Sender<Cmd>> {
+ TX.get().ok_or_else(|| anyhow::anyhow!("libei worker not started; call ensure_started() first"))
+}
+
+fn restore_token_path() -> Option<PathBuf> {
+ let base = dirs::config_dir()?;
+ Some(base.join("cua-driver").join("libei.token"))
+}
+
+fn read_restore_token() -> Option<String> {
+ let path = restore_token_path()?;
+ std::fs::read_to_string(path).ok().map(|s| s.trim().to_string()).filter(|s| !s.is_empty())
+}
+
+fn write_restore_token(token: &str) -> anyhow::Result<()> {
+ let path = restore_token_path()
+ .ok_or_else(|| anyhow::anyhow!("no config dir available for libei restore_token"))?;
+ if let Some(parent) = path.parent() {
+ std::fs::create_dir_all(parent).map_err(|e| {
+ anyhow::anyhow!("failed to create {} for restore_token: {e}", parent.display())
+ })?;
+ }
+ std::fs::write(&path, token)
+ .map_err(|e| anyhow::anyhow!("failed to write libei restore_token to {}: {e}", path.display()))
+}
+
+/// Spawn the libei worker thread (idempotent — safe to call from every
+/// MCP tool invocation; subsequent calls are no-ops).
+pub fn ensure_started() -> anyhow::Result<()> {
+ TX.get_or_init(|| {
+ let (tx, rx) = bounded::<Cmd>(64);
+ thread::Builder::new()
+ .name("cua-libei-worker".into())
+ .spawn(move || {
+ if let Err(e) = worker(rx) {
+ tracing::warn!("cua-libei-worker exited with error: {e}");
+ }
+ })
+ .expect("spawn cua-libei-worker thread");
+ tx
+ });
+ Ok(())
+}
+
+// ── Public API ───────────────────────────────────────────────────────────
+
+/// Move the cursor to absolute (x, y) within the announced device region,
+/// then press + release `button`. Blocks until the EIS server has
+/// received the event sequence.
+pub fn click(x: f64, y: f64, button: Button) -> anyhow::Result<()> {
+ ensure_started()?;
+ let (tx_r, rx_r) = bounded(1);
+ tx()?.send(Cmd::Click { x, y, button, reply: tx_r })
+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+}
+
+/// Move the cursor to absolute (x, y) inside the device region (no
+/// button press).
+pub fn move_absolute(x: f64, y: f64) -> anyhow::Result<()> {
+ ensure_started()?;
+ let (tx_r, rx_r) = bounded(1);
+ tx()?.send(Cmd::MoveAbsolute { x, y, reply: tx_r })
+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+}
+
+/// Scroll by (dx, dy) logical units. Positive y scrolls down.
+pub fn scroll(dx: f64, dy: f64) -> anyhow::Result<()> {
+ ensure_started()?;
+ let (tx_r, rx_r) = bounded(1);
+ tx()?.send(Cmd::Scroll { dx, dy, reply: tx_r })
+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+}
+
+/// Inject a UTF-8 string via the `ei_text` interface (libei 1.6+).
+/// Bypasses keycode reverse-mapping — works correctly for any layout.
+pub fn type_text(text: &str) -> anyhow::Result<()> {
+ ensure_started()?;
+ let (tx_r, rx_r) = bounded(1);
+ tx()?.send(Cmd::TypeText { text: text.to_string(), reply: tx_r })
+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+}
+
+/// Press + release a key by evdev code (e.g. `KEY_ENTER` = 28). For
+/// modifier combos use the keyboard interface's modifier state explicitly.
+pub fn press_key(keycode: u32) -> anyhow::Result<()> {
+ ensure_started()?;
+ let (tx_r, rx_r) = bounded(1);
+ tx()?.send(Cmd::PressKey { keycode, reply: tx_r })
+ .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?;
+ rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))?
+}
+
+/// Cleanly stop the worker thread.
+pub fn shutdown() {
+ if let Some(tx) = TX.get() {
+ let _ = tx.send(Cmd::Shutdown);
+ }
+}
+
+// ── Worker thread ────────────────────────────────────────────────────────
+//
+// The worker owns the `reis::ei::Context`, the per-seat device map, and
+// the negotiated input region. It runs a calloop event loop that handles
+// both the EIS protocol and the inbound command channel.
+
+fn worker(rx: Receiver<Cmd>) -> anyhow::Result<()> {
+ // Phase 1 — acquire an EIS context.
+ let context = open_eis_context()
+ .map_err(|e| anyhow::anyhow!("failed to obtain EIS connection: {e}"))?;
+
+ // Phase 2 — handshake. The reis API requires we call context.handshake()
+ // and flush before the event loop starts polling.
+ let _handshake = context.handshake();
+ let _ = context.flush();
+
+ // Phase 3 — run the calloop event loop. We use a calloop Generic source
+ // wrapping the ei::Context's underlying socket so the loop wakes up on
+ // EIS messages. The command channel is polled at the top of each loop
+ // iteration.
+ run_calloop(context, rx)
+}
+
+/// Open the EIS context. Fast path: $LIBEI_SOCKET (cua-compositor /
+/// inject mode). Fallback: ashpd portal RemoteDesktop handshake.
+fn open_eis_context() -> anyhow::Result<reis::ei::Context> {
+ if let Some(ctx) = reis::ei::Context::connect_to_env()
+ .map_err(|e| anyhow::anyhow!("env ei socket open failed: {e}"))?
+ {
+ return Ok(ctx);
+ }
+
+ // Portal RemoteDesktop fallback.
+ use ashpd::desktop::{
+ remote_desktop::{
+ ConnectToEISOptions, DeviceType, RemoteDesktop, SelectDevicesOptions, StartOptions,
+ },
+ CreateSessionOptions, PersistMode,
+ };
+ use ashpd::enumflags2::BitFlags;
+ use std::os::unix::net::UnixStream;
+
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?;
+
+ let fd = rt.block_on(async {
+ let proxy = RemoteDesktop::new()
+ .await
+ .map_err(|e| anyhow::anyhow!("portal RemoteDesktop proxy unreachable: {e}. Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde."))?;
+ let session = proxy
+ .create_session(CreateSessionOptions::default())
+ .await
+ .map_err(|e| anyhow::anyhow!("portal create_session failed: {e}"))?;
+
+ let mut select_opts = SelectDevicesOptions::default()
+ .set_devices(BitFlags::<DeviceType>::from(DeviceType::Keyboard) | DeviceType::Pointer)
+ .set_persist_mode(PersistMode::Application);
+ if let Some(tok) = read_restore_token() {
+ select_opts = select_opts.set_restore_token(Some(tok.as_str()));
+ }
+
+ proxy
+ .select_devices(&session, select_opts)
+ .await
+ .map_err(|e| anyhow::anyhow!("portal select_devices failed: {e}"))?
+ .response()
+ .map_err(|e| anyhow::anyhow!("portal select_devices response error: {e}"))?;
+
+ let started = proxy
+ .start(&session, None, StartOptions::default())
+ .await
+ .map_err(|e| anyhow::anyhow!("portal start failed: {e}"))?
+ .response()
+ .map_err(|e| anyhow::anyhow!("portal start response error (user denied?): {e}"))?;
+
+ // Best-effort persist of the restore_token. Without this, every
+ // process restart re-prompts the user for consent — with it, ashpd
+ // 0.13's PersistMode::Application + the stored token together
+ // skip the dialog for the lifetime of the persistence grant
+ // (typically per login session on GNOME, indefinite on KDE).
+ if let Some(tok) = started.restore_token() {
+ let _ = write_restore_token(tok);
+ }
+
+ let fd = proxy
+ .connect_to_eis(&session, ConnectToEISOptions::default())
+ .await
+ .map_err(|e| anyhow::anyhow!("portal connect_to_eis failed: {e}"))?;
+ anyhow::Ok(fd)
+ })?;
+
+ let stream = UnixStream::from(fd);
+ let ctx = reis::ei::Context::new(stream)
+ .map_err(|e| anyhow::anyhow!("reis ei::Context::new failed: {e}"))?;
+ Ok(ctx)
+}
+
+// ── calloop dispatch ────────────────────────────────────────────────────
+//
+// The EIS protocol is event-driven and non-trivial — handshake → connection
+// → seat → device → frame negotiation. We model the worker as a calloop
+// EventLoop driving the ei::Context Generic source, with a separate
+// channel source for inbound Cmd messages.
+//
+// For v1 we implement the handshake + seat-bind dance and provide
+// click/move/type via the negotiated pointer/keyboard/text interfaces.
+// Region selection picks the first announced ei_device::Region.
+
+fn run_calloop(context: reis::ei::Context, rx: Receiver<Cmd>) -> anyhow::Result<()> {
+ use calloop::{generic::Generic, EventLoop, Interest, Mode, PostAction};
+
+ let mut event_loop: EventLoop<EisState> = EventLoop::try_new()
+ .map_err(|e| anyhow::anyhow!("calloop EventLoop::try_new failed: {e}"))?;
+ let handle = event_loop.handle();
+
+ // EIS protocol source.
+ let ctx_source = Generic::new(context, Interest::READ, Mode::Level);
+ handle
+ .insert_source(ctx_source, |_event, ctx, state: &mut EisState| {
+ state.handle_eis_readable(unsafe { ctx.get_mut() })
+ })
+ .map_err(|e| anyhow::anyhow!("calloop insert_source(eis) failed: {e}"))?;
+
+ let mut state = EisState::default();
+
+ // Command channel: drain at the top of each loop iteration with a
+ // bounded timeout so the EIS source stays responsive.
+ loop {
+ // Process pending commands.
+ match rx.try_recv() {
+ Ok(Cmd::Shutdown) => break,
+ Ok(cmd) => state.handle_command(cmd),
+ Err(crossbeam_channel::TryRecvError::Empty) => {}
+ Err(crossbeam_channel::TryRecvError::Disconnected) => break,
+ }
+
+ event_loop
+ .dispatch(Some(std::time::Duration::from_millis(20)), &mut state)
+ .map_err(|e| anyhow::anyhow!("calloop dispatch failed: {e}"))?;
+
+ // The handle_command path may have queued frame() requests; flush
+ // them once per loop iteration so the EIS server sees them.
+ if let Some(ctx) = state.context.as_ref() {
+ let _ = ctx.flush();
+ }
+ }
+
+ Ok(())
+}
+
+#[derive(Default)]
+struct EisState {
+ context: Option<reis::ei::Context>,
+ seats: HashMap<reis::ei::Seat, SeatData>,
+ devices: HashMap<reis::ei::Device, DeviceData>,
+ sequence: u32,
+ last_serial: u32,
+}
+
+#[derive(Default)]
+struct SeatData {
+ capabilities: HashMap<String, u64>,
+}
+
+/// One announced ei_device::Region — a single monitor's pixel rectangle
+/// inside the seat's logical coordinate space. Multi-monitor sessions
+/// announce one Region per output; we route each absolute (x, y) to the
+/// region containing it so the cursor lands on the correct monitor.
+#[derive(Clone, Copy, Debug)]
+struct RegionRect {
+ offset_x: f32,
+ offset_y: f32,
+ width: f32,
+ height: f32,
+}
+
+impl RegionRect {
+ fn contains(&self, x: f64, y: f64) -> bool {
+ let xf = x as f32;
+ let yf = y as f32;
+ xf >= self.offset_x
+ && xf < self.offset_x + self.width
+ && yf >= self.offset_y
+ && yf < self.offset_y + self.height
+ }
+}
+
+#[derive(Default)]
+struct DeviceData {
+ device_type: Option<reis::ei::device::DeviceType>,
+ /// Every Region the device announces between its creation and its
+ /// first Done event. Empty until the EIS server has sent at least
+ /// one Region.
+ regions: Vec<RegionRect>,
+ interfaces: HashMap<String, reis::Object>,
+}
+
+impl EisState {
+ fn handle_eis_readable(
+ &mut self,
+ context: &mut reis::ei::Context,
+ ) -> std::io::Result<calloop::PostAction> {
+ use reis::PendingRequestResult;
+ if self.context.is_none() {
+ self.context = Some(context.clone());
+ }
+ if context.read().is_err() {
+ return Ok(calloop::PostAction::Remove);
+ }
+ while let Some(result) = context.pending_event() {
+ let request = match result {
+ PendingRequestResult::Request(r) => r,
+ _ => continue,
+ };
+ self.dispatch_eis_event(request);
+ }
+ let _ = context.flush();
+ Ok(calloop::PostAction::Continue)
+ }
+
+ fn dispatch_eis_event(&mut self, event: reis::ei::Event) {
+ use reis::ei::Event;
+ match event {
+ Event::Handshake(handshake, ev) => {
+ if let reis::ei::handshake::Event::HandshakeVersion { .. } = ev {
+ handshake.handshake_version(1);
+ handshake.name("cua-driver");
+ handshake.context_type(reis::ei::handshake::ContextType::Sender);
+ for (iface, ver) in [
+ ("ei_callback", 1u32),
+ ("ei_connection", 1),
+ ("ei_seat", 1),
+ ("ei_device", 2),
+ ("ei_pointer", 1),
+ ("ei_pointer_absolute", 1),
+ ("ei_button", 1),
+ ("ei_scroll", 1),
+ ("ei_keyboard", 1),
+ ("ei_text", 1),
+ ] {
+ handshake.interface_version(iface, ver);
+ }
+ handshake.finish();
+ }
+ }
+ Event::Connection(_conn, ev) => {
+ if let reis::ei::connection::Event::Seat { seat } = ev {
+ self.seats.insert(seat, SeatData::default());
+ }
+ }
+ Event::Seat(seat, ev) => {
+ let data = self.seats.entry(seat.clone()).or_default();
+ match ev {
+ reis::ei::seat::Event::Capability { mask, interface } => {
+ data.capabilities.insert(interface, mask);
+ }
+ reis::ei::seat::Event::Done => {
+ // Bind every input capability the seat advertises
+ // (pointer / pointer_absolute / button / scroll /
+ // keyboard / text). The server then announces
+ // matching devices.
+ let mut mask = 0u64;
+ for k in [
+ "ei_pointer",
+ "ei_pointer_absolute",
+ "ei_button",
+ "ei_scroll",
+ "ei_keyboard",
+ "ei_text",
+ ] {
+ if let Some(m) = data.capabilities.get(k) {
+ mask |= *m;
+ }
+ }
+ if mask != 0 {
+ seat.bind(mask);
+ }
+ }
+ reis::ei::seat::Event::Device { device } => {
+ self.devices.insert(device, DeviceData::default());
+ }
+ _ => {}
+ }
+ }
+ Event::Device(device, ev) => {
+ let data = self.devices.entry(device.clone()).or_default();
+ match ev {
+ reis::ei::device::Event::DeviceType { device_type } => {
+ data.device_type = Some(device_type);
+ }
+ reis::ei::device::Event::Interface { object } => {
+ data.interfaces.insert(object.interface().to_owned(), object);
+ }
+ reis::ei::device::Event::Region { offset_x, offset_y, width, hight, scale: _ } => {
+ // reis 0.7 keeps the misspelled "hight" field name
+ // for ABI compatibility — see the protocol comment.
+ if width > 0 && hight > 0 {
+ data.regions.push(RegionRect {
+ offset_x: offset_x as f32,
+ offset_y: offset_y as f32,
+ width: width as f32,
+ height: hight as f32,
+ });
+ }
+ }
+ reis::ei::device::Event::Resumed { serial } => {
+ self.last_serial = serial;
+ }
+ _ => {}
+ }
+ }
+ _ => {}
+ }
+ }
+
+ fn handle_command(&mut self, cmd: Cmd) {
+ let result = self.run_command(&cmd);
+ // Send the reply on whichever channel this command carries.
+ match cmd {
+ Cmd::Click { reply, .. } => { let _ = reply.send(result); }
+ Cmd::MoveAbsolute { reply, .. } => { let _ = reply.send(result); }
+ Cmd::Scroll { reply, .. } => { let _ = reply.send(result); }
+ Cmd::TypeText { reply, .. } => { let _ = reply.send(result); }
+ Cmd::PressKey { reply, .. } => { let _ = reply.send(result); }
+ Cmd::Shutdown => {}
+ }
+ }
+
+ fn run_command(&mut self, cmd: &Cmd) -> anyhow::Result<()> {
+ match cmd {
+ Cmd::Click { x, y, button, .. } => {
+ let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?;
+ if let Some(ptr_abs) = device_interface::<reis::ei::PointerAbsolute>(&self.devices, &device) {
+ device.start_emulating(self.sequence, self.last_serial);
+ self.sequence = self.sequence.wrapping_add(1);
+ ptr_abs.motion_absolute(rel_x, rel_y);
+ }
+ if let Some(btn) = device_interface::<reis::ei::Button>(&self.devices, &device) {
+ btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press);
+ device.frame(self.last_serial, 0);
+ btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released);
+ device.frame(self.last_serial, 1);
+ }
+ device.stop_emulating(self.last_serial);
+ }
+ Cmd::MoveAbsolute { x, y, .. } => {
+ let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?;
+ if let Some(ptr_abs) = device_interface::<reis::ei::PointerAbsolute>(&self.devices, &device) {
+ device.start_emulating(self.sequence, self.last_serial);
+ self.sequence = self.sequence.wrapping_add(1);
+ ptr_abs.motion_absolute(rel_x, rel_y);
+ device.frame(self.last_serial, 0);
+ device.stop_emulating(self.last_serial);
+ }
+ }
+ Cmd::Scroll { dx, dy, .. } => {
+ let device = self.any_pointer_device()
+ .ok_or_else(|| anyhow::anyhow!("no EIS pointer device negotiated yet — wait for handshake"))?;
+ if let Some(scroll) = device_interface::<reis::ei::Scroll>(&self.devices, &device) {
+ device.start_emulating(self.sequence, self.last_serial);
+ self.sequence = self.sequence.wrapping_add(1);
+ scroll.scroll(*dx as f32, *dy as f32);
+ device.frame(self.last_serial, 0);
+ device.stop_emulating(self.last_serial);
+ }
+ }
+ Cmd::TypeText { text, .. } => {
+ let device = self.any_pointer_device()
+ .ok_or_else(|| anyhow::anyhow!("no EIS device negotiated yet — wait for handshake"))?;
+ if let Some(text_iface) = device_interface::<reis::ei::Text>(&self.devices, &device) {
+ device.start_emulating(self.sequence, self.last_serial);
+ self.sequence = self.sequence.wrapping_add(1);
+ text_iface.utf8(text);
+ device.frame(self.last_serial, 0);
+ device.stop_emulating(self.last_serial);
+ }
+ }
+ Cmd::PressKey { keycode, .. } => {
+ let device = self.any_pointer_device()
+ .ok_or_else(|| anyhow::anyhow!("no EIS device negotiated yet — wait for handshake"))?;
+ if let Some(kb) = device_interface::<reis::ei::Keyboard>(&self.devices, &device) {
+ device.start_emulating(self.sequence, self.last_serial);
+ self.sequence = self.sequence.wrapping_add(1);
+ kb.key(*keycode, reis::ei::keyboard::KeyState::Press);
+ device.frame(self.last_serial, 0);
+ kb.key(*keycode, reis::ei::keyboard::KeyState::Released);
+ device.frame(self.last_serial, 1);
+ device.stop_emulating(self.last_serial);
+ }
+ }
+ Cmd::Shutdown => {}
+ }
+ if let Some(ctx) = self.context.as_ref() {
+ let _ = ctx.flush();
+ }
+ Ok(())
+ }
+
+ /// Pick a pointer device whose announced Region contains `(x, y)`,
+ /// and translate the absolute screen coordinates into the device's
+ /// region-local pixel coordinates.
+ ///
+ /// Multi-monitor seats announce one Region per output; routing each
+ /// (x, y) by containment is what lets a click on monitor #2 actually
+ /// land on monitor #2. When no device's region contains the point —
+ /// e.g. coords just past the edge of an output, or a session with
+ /// a single device that hasn't announced any regions yet — we fall
+ /// back to the first pointer device with the raw coordinates, which
+ /// matches the pre-multi-monitor behaviour.
+ fn pointer_device_for(&self, x: f64, y: f64) -> anyhow::Result<(reis::ei::Device, f32, f32)> {
+ let is_pointer = |data: &DeviceData| matches!(
+ data.device_type,
+ Some(reis::ei::device::DeviceType::Virtual) | Some(reis::ei::device::DeviceType::Physical)
+ );
+ for (device, data) in self.devices.iter() {
+ if !is_pointer(data) {
+ continue;
+ }
+ for region in &data.regions {
+ if region.contains(x, y) {
+ let rel_x = x as f32 - region.offset_x;
+ let rel_y = y as f32 - region.offset_y;
+ return Ok((device.clone(), rel_x, rel_y));
+ }
+ }
+ }
+ // Fallback: first pointer device, raw coords (single-region behaviour).
+ let device = self.any_pointer_device()
+ .ok_or_else(|| anyhow::anyhow!("no EIS pointer device negotiated yet — wait for handshake"))?;
+ Ok((device, x as f32, y as f32))
+ }
+
+ fn any_pointer_device(&self) -> Option<reis::ei::Device> {
+ self.devices.iter()
+ .find(|(_, data)| matches!(
+ data.device_type,
+ Some(reis::ei::device::DeviceType::Virtual) | Some(reis::ei::device::DeviceType::Physical)
+ ))
+ .map(|(d, _)| d.clone())
+ }
+}
+
+fn device_interface<T: reis::Interface>(
+ devices: &HashMap<reis::ei::Device, DeviceData>,
+ device: &reis::ei::Device,
+) -> Option<T> {
+ devices.get(device)?
+ .interfaces
+ .get(T::NAME)?
+ .clone()
+ .downcast()
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
index 5c7d0bf9ad..da9cf1457d 100644
--- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs
@@ -1,16 +1,36 @@
//! Native-Wayland backend.
//!
//! Used when running under a Wayland compositor with no X11 (WAYLAND_DISPLAY
-//! set, DISPLAY unset). The first slice enumerates toplevels via
-//! `zwlr_foreign_toplevel_manager_v1` (wlroots: labwc, sway, …) so
-//! `list_windows` works on native Wayland. Capture and input slices land next.
+//! set, DISPLAY unset). Enumerates toplevels via
+//! `zwlr_foreign_toplevel_manager_v1`, captures per-output screenshots via
+//! `zwlr_screencopy_manager_v1` + `wl_shm` (native — `grim` remains a
+//! fallback), and synthesises pointer / scroll / drag input via
+//! `zwlr_virtual_pointer_v1`. Per-window image capture is deferred until
+//! `ext-foreign-toplevel-image-capture-source-v1` lands in
+//! `wayland-protocols-wlr`; until then `screenshot_window_dispatch` returns a
+//! typed error on pure Wayland.
+
+pub mod persistent_vptr;
+pub mod portal_screenshot;
+pub mod overlay;
+pub mod ext_screencopy;
+pub mod portal_screencast;
+pub mod libei;
use std::collections::HashMap;
use wayland_client::{
event_created_child,
- protocol::{wl_output::{self, WlOutput}, wl_pointer::ButtonState, wl_registry, wl_seat::WlSeat},
- Connection, Dispatch, Proxy, QueueHandle,
+ protocol::{
+ wl_buffer::WlBuffer,
+ wl_output::{self, WlOutput},
+ wl_pointer::{Axis, AxisSource, ButtonState},
+ wl_registry,
+ wl_seat::WlSeat,
+ wl_shm::{self, WlShm},
+ wl_shm_pool::WlShmPool,
+ },
+ Connection, Dispatch, Proxy, QueueHandle, WEnum,
};
use wayland_protocols_wlr::foreign_toplevel::v1::client::{
zwlr_foreign_toplevel_handle_v1::{self as ftl_handle, ZwlrForeignToplevelHandleV1},
@@ -18,6 +38,10 @@ use wayland_protocols_wlr::foreign_toplevel::v1::client::{
self as ftl_manager, ZwlrForeignToplevelManagerV1, EVT_TOPLEVEL_OPCODE,
},
};
+use wayland_protocols_wlr::screencopy::v1::client::{
+ zwlr_screencopy_frame_v1::{self as scrcopy_frame, ZwlrScreencopyFrameV1},
+ zwlr_screencopy_manager_v1::ZwlrScreencopyManagerV1,
+};
use wayland_protocols_wlr::virtual_pointer::v1::client::{
zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,
zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,
@@ -34,12 +58,14 @@ pub const ENABLE_WAYLAND_ENV: &str = "CUA_DRIVER_RS_ENABLE_WAYLAND";
/// Whether the user has opted into the experimental native-Wayland backend.
///
-/// The Wayland backend is incomplete (toplevel enumeration + virtual-pointer /
-/// virtual-keyboard input via the wlroots protocols; capture and AT-SPI parity
-/// are still landing), so it stays OFF by default and a pure-Wayland session is
-/// reported as unsupported unless the user explicitly sets
-/// `CUA_DRIVER_RS_ENABLE_WAYLAND=1`. Any value other than empty / `0` / `false`
-/// enables it.
+/// The Wayland backend covers toplevel enumeration, per-output capture
+/// (native screencopy + a `grim` fallback), and virtual-pointer /
+/// virtual-keyboard input via the wlroots protocols. Per-window image
+/// capture still depends on the staging `ext-image-copy-capture-v1`
+/// protocol, so the backend stays OFF by default and a pure-Wayland
+/// session is reported as unsupported unless the user explicitly sets
+/// `CUA_DRIVER_RS_ENABLE_WAYLAND=1`. Any value other than empty / `0` /
+/// `false` enables it.
pub fn wayland_enabled() -> bool {
match std::env::var(ENABLE_WAYLAND_ENV) {
Ok(v) => {
@@ -133,6 +159,19 @@ struct Toplevel {
closed: bool,
}
+/// Per-capture in-flight state populated by the screencopy frame Dispatch.
+#[derive(Default)]
+struct CaptureState {
+ /// wl_shm format code (Argb8888 / Xrgb8888 / …).
+ format: Option<u32>,
+ width: u32,
+ height: u32,
+ stride: u32,
+ y_invert: bool,
+ ready: bool,
+ failed: bool,
+}
+
#[derive(Default)]
struct State {
manager: Option<ZwlrForeignToplevelManagerV1>,
@@ -144,8 +183,13 @@ struct State {
// Virtual-pointer manager + output dimensions, so `click` can land a real
// button press at the output centre (over the just-activated window).
vptr_manager: Option<ZwlrVirtualPointerManagerV1>,
+ output: Option<WlOutput>,
output_w: u32,
output_h: u32,
+ // Native screencopy capture state.
+ scrcopy_manager: Option<ZwlrScreencopyManagerV1>,
+ shm: Option<WlShm>,
+ capture: CaptureState,
}
impl Dispatch<wl_registry::WlRegistry, ()> for State {
@@ -168,7 +212,15 @@ impl Dispatch<wl_registry::WlRegistry, ()> for State {
state.vptr_manager =
Some(registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(name, version.min(2), qh, ()));
} else if interface == WlOutput::interface().name {
- registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ());
+ let out = registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ());
+ if state.output.is_none() {
+ state.output = Some(out);
+ }
+ } else if interface == ZwlrScreencopyManagerV1::interface().name {
+ state.scrcopy_manager =
+ Some(registry.bind::<ZwlrScreencopyManagerV1, _, _>(name, version.min(3), qh, ()));
+ } else if interface == WlShm::interface().name {
+ state.shm = Some(registry.bind::<WlShm, _, _>(name, version.min(1), qh, ()));
}
}
}
@@ -229,6 +281,91 @@ impl Dispatch<ZwlrVirtualPointerV1, ()> for State {
}
}
+impl Dispatch<WlShm, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &WlShm,
+ _: wl_shm::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ // wl_shm advertises supported formats via `format` events; we don't
+ // need to track them — screencopy tells us exactly which format to use
+ // for the frame buffer.
+ }
+}
+
+impl Dispatch<WlShmPool, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &WlShmPool,
+ _: wayland_client::protocol::wl_shm_pool::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ }
+}
+
+impl Dispatch<WlBuffer, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &WlBuffer,
+ _: wayland_client::protocol::wl_buffer::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ }
+}
+
+impl Dispatch<ZwlrScreencopyManagerV1, ()> for State {
+ fn event(
+ _: &mut Self,
+ _: &ZwlrScreencopyManagerV1,
+ _: <ZwlrScreencopyManagerV1 as Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ }
+}
+
+impl Dispatch<ZwlrScreencopyFrameV1, ()> for State {
+ fn event(
+ state: &mut Self,
+ _: &ZwlrScreencopyFrameV1,
+ event: scrcopy_frame::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ match event {
+ scrcopy_frame::Event::Buffer { format, width, height, stride } => {
+ if let WEnum::Value(fmt) = format {
+ state.capture.format = Some(fmt as u32);
+ }
+ state.capture.width = width;
+ state.capture.height = height;
+ state.capture.stride = stride;
+ }
+ scrcopy_frame::Event::Flags { flags } => {
+ if let WEnum::Value(f) = flags {
+ state.capture.y_invert = f.contains(scrcopy_frame::Flags::YInvert);
+ }
+ }
+ scrcopy_frame::Event::Ready { .. } => {
+ state.capture.ready = true;
+ }
+ scrcopy_frame::Event::Failed => {
+ state.capture.failed = true;
+ }
+ _ => {}
+ }
+ }
+}
+
impl Dispatch<ZwlrForeignToplevelManagerV1, ()> for State {
fn event(
_state: &mut Self,
@@ -305,12 +442,27 @@ pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> {
Ok(out)
}
-/// Capture the Wayland output as PNG bytes via `grim` (the wlroots screenshot
-/// tool — wlr-screencopy under the hood, works on labwc/sway). This mirrors the
-/// X11 backend shelling out to `import`/`xwd`. foreign-toplevel exposes no
-/// per-window geometry, so this captures the whole output; that is sufficient
-/// for `get_window_state`'s vision payload.
+// ── Capture (native screencopy + grim fallback) ──────────────────────────────
+
+/// Capture the Wayland output as PNG bytes via `zwlr_screencopy_manager_v1`.
+///
+/// Binds the screencopy manager plus `wl_shm`, asks the compositor to copy the
+/// next frame of the first advertised output into a shm buffer, channel-swaps
+/// from the compositor's pixel format to RGBA, and encodes a PNG via the
+/// existing `image` crate. Falls back to shelling out to `grim` when the
+/// screencopy manager or `wl_shm` is unavailable so users on lighter wlroots
+/// builds stay supported.
pub fn screenshot_bytes() -> anyhow::Result<Vec<u8>> {
+ match capture_via_screencopy() {
+ Ok(bytes) => return Ok(bytes),
+ Err(e) => tracing::warn!("native screencopy failed, falling back to grim: {e}"),
+ }
+ capture_via_grim()
+}
+
+/// Shell out to `grim -t png -` — the wlroots reference screenshot tool. Kept
+/// as the last-resort fallback for compositors that hide screencopy.
+fn capture_via_grim() -> anyhow::Result<Vec<u8>> {
let out = std::process::Command::new("grim").args(["-t", "png", "-"]).output()?;
if !out.status.success() {
anyhow::bail!("grim failed: {}", String::from_utf8_lossy(&out.stderr));
@@ -321,7 +473,202 @@ pub fn screenshot_bytes() -> anyhow::Result<Vec<u8>> {
Ok(out.stdout)
}
-/// Capture dispatcher: native Wayland (grim) when applicable, else X11.
+/// Native screencopy path: bind manager + shm, allocate an anon mmap buffer,
+/// request a copy, wait for Ready, swap channels, encode PNG. Returns an error
+/// if any global is missing or the compositor flags the capture as failed.
+fn capture_via_screencopy() -> anyhow::Result<Vec<u8>> {
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<State>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+
+ let mut state = State::default();
+ queue.roundtrip(&mut state)?;
+ queue.roundtrip(&mut state)?; // outputs report their Mode
+
+ let manager = state
+ .scrcopy_manager
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose zwlr_screencopy_manager_v1"))?;
+ let shm = state
+ .shm
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_shm"))?;
+ let output = state
+ .output
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_output to capture"))?;
+
+ let frame = manager.capture_output(0, &output, &qh, ());
+ // Drain Buffer / Flags events; spin until Ready or Failed (or timeout).
+ let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
+ let mut buffer: Option<WlBuffer> = None;
+ let mut pool: Option<WlShmPool> = None;
+ let mut mmap_ptr: *mut libc::c_void = std::ptr::null_mut();
+ let mut mmap_len: usize = 0;
+ let mut fd: i32 = -1;
+
+ loop {
+ queue.roundtrip(&mut state)?;
+ if state.capture.failed {
+ anyhow::bail!("compositor signalled screencopy failure");
+ }
+ if state.capture.ready {
+ break;
+ }
+ // Once we know the buffer params, allocate + send copy exactly once.
+ if buffer.is_none()
+ && state.capture.format.is_some()
+ && state.capture.stride > 0
+ && state.capture.height > 0
+ {
+ let size = (state.capture.stride as usize)
+ .checked_mul(state.capture.height as usize)
+ .ok_or_else(|| anyhow::anyhow!("screencopy buffer size overflow"))?;
+ let (anon_fd, p) = anon_shm(size)?;
+ fd = anon_fd;
+ mmap_ptr = p;
+ mmap_len = size;
+ use std::os::fd::AsFd as _;
+ let pool_fd = unsafe { borrowed_fd(fd) };
+ let p = shm.create_pool(pool_fd.as_fd(), size as i32, &qh, ());
+ let fmt_raw = state.capture.format.unwrap();
+ let fmt: wl_shm::Format = match wl_shm::Format::try_from(fmt_raw) {
+ Ok(f) => f,
+ Err(_) => {
+ cleanup_mmap(mmap_ptr, mmap_len, fd);
+ anyhow::bail!("compositor advertised unsupported wl_shm format {fmt_raw:#x}");
+ }
+ };
+ let b = p.create_buffer(
+ 0,
+ state.capture.width as i32,
+ state.capture.height as i32,
+ state.capture.stride as i32,
+ fmt,
+ &qh,
+ (),
+ );
+ frame.copy(&b);
+ buffer = Some(b);
+ pool = Some(p);
+ }
+ if std::time::Instant::now() >= deadline {
+ cleanup_mmap(mmap_ptr, mmap_len, fd);
+ anyhow::bail!("screencopy timed out waiting for frame");
+ }
+ }
+
+ let result = (|| -> anyhow::Result<Vec<u8>> {
+ let w = state.capture.width;
+ let h = state.capture.height;
+ let stride = state.capture.stride as usize;
+ let format = state.capture.format.unwrap_or(0);
+ if mmap_ptr.is_null() || mmap_len == 0 {
+ anyhow::bail!("screencopy ready without a backing buffer");
+ }
+ let raw = unsafe { std::slice::from_raw_parts(mmap_ptr as *const u8, mmap_len) };
+ let mut rgba = Vec::with_capacity((w as usize) * (h as usize) * 4);
+ for row in 0..(h as usize) {
+ let src_row = if state.capture.y_invert { (h as usize) - 1 - row } else { row };
+ let base = src_row * stride;
+ for col in 0..(w as usize) {
+ let px = &raw[base + col * 4..base + col * 4 + 4];
+ let (r, g, b, a) = match wl_shm::Format::try_from(format).ok() {
+ // Argb8888 / Xrgb8888 over wl_shm are little-endian BGRA / BGRX.
+ Some(wl_shm::Format::Argb8888) => (px[2], px[1], px[0], px[3]),
+ Some(wl_shm::Format::Xrgb8888) => (px[2], px[1], px[0], 255),
+ Some(wl_shm::Format::Abgr8888) => (px[0], px[1], px[2], px[3]),
+ Some(wl_shm::Format::Xbgr8888) => (px[0], px[1], px[2], 255),
+ _ => (px[2], px[1], px[0], px[3]),
+ };
+ rgba.extend_from_slice(&[r, g, b, a]);
+ }
+ }
+ cua_driver_core::image_utils::encode_rgba_to_png(&rgba, w, h)
+ })();
+
+ // Always tear down regardless of result.
+ if let Some(b) = buffer {
+ b.destroy();
+ }
+ if let Some(p) = pool {
+ p.destroy();
+ }
+ frame.destroy();
+ let _ = queue.roundtrip(&mut state);
+ cleanup_mmap(mmap_ptr, mmap_len, fd);
+
+ result
+}
+
+/// Allocate an anonymous shared-memory file of `size` bytes and mmap it RW.
+/// Returns the raw fd and the mmap pointer; the caller is responsible for
+/// passing both to [`cleanup_mmap`] when done.
+pub(crate) fn anon_shm(size: usize) -> anyhow::Result<(i32, *mut libc::c_void)> {
+ // memfd_create is Linux-only and is the cleanest path; fall back to
+ // shm_open if memfd isn't available for any reason.
+ let name = b"cua-scrcopy\0";
+ let fd = unsafe { libc::memfd_create(name.as_ptr() as *const libc::c_char, libc::MFD_CLOEXEC) };
+ if fd < 0 {
+ return Err(anyhow::anyhow!("memfd_create failed: {}", std::io::Error::last_os_error()));
+ }
+ let rc = unsafe { libc::ftruncate(fd, size as libc::off_t) };
+ if rc != 0 {
+ let err = std::io::Error::last_os_error();
+ unsafe { libc::close(fd) };
+ return Err(anyhow::anyhow!("ftruncate failed: {err}"));
+ }
+ let p = unsafe {
+ libc::mmap(
+ std::ptr::null_mut(),
+ size,
+ libc::PROT_READ | libc::PROT_WRITE,
+ libc::MAP_SHARED,
+ fd,
+ 0,
+ )
+ };
+ if p == libc::MAP_FAILED {
+ let err = std::io::Error::last_os_error();
+ unsafe { libc::close(fd) };
+ return Err(anyhow::anyhow!("mmap failed: {err}"));
+ }
+ Ok((fd, p))
+}
+
+/// Unmap and close the screencopy backing buffer; safe to call with the
+/// sentinel values left from a never-allocated buffer.
+pub(crate) fn cleanup_mmap(ptr: *mut libc::c_void, len: usize, fd: i32) {
+ if !ptr.is_null() && len > 0 {
+ unsafe { libc::munmap(ptr, len) };
+ }
+ if fd >= 0 {
+ unsafe { libc::close(fd) };
+ }
+}
+
+/// Borrow a raw fd as an `OwnedFd` for wl_shm.create_pool. The pool keeps
+/// its own reference; we close our copy via [`cleanup_mmap`].
+///
+/// SAFETY: caller must guarantee `fd` is a valid open file descriptor.
+/// `libc::dup` may fail (returning -1, errno set), in which case we panic
+/// instead of constructing an `OwnedFd` from -1 (which would have UB on
+/// drop). Callers that need fallible behaviour should use the dup syscall
+/// directly and check the result before wrapping.
+pub(crate) unsafe fn borrowed_fd(fd: i32) -> std::os::fd::OwnedFd {
+ use std::os::fd::FromRawFd;
+ let dup = libc::dup(fd);
+ if dup < 0 {
+ let err = std::io::Error::last_os_error();
+ panic!("dup({fd}) failed: {err}");
+ }
+ std::os::fd::OwnedFd::from_raw_fd(dup)
+}
+
+/// Capture dispatcher: native Wayland (screencopy with grim fallback) when
+/// applicable, else X11. Mirrors `screenshot_window_dispatch` for the
+/// output-level path used by `get_window_state`'s vision payload.
pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {
if is_wayland() {
screenshot_bytes()
@@ -330,61 +677,302 @@ pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {
}
}
-/// Click a native Wayland toplevel identified by its `window_id` (the
-/// foreign-toplevel protocol id from `list_windows`). Wayland forbids a client
-/// from knowing another window's on-screen geometry, so we cannot map the
-/// caller's window-local x/y to a global pointer position. Instead, the
-/// focused-input model is two steps:
-/// 1. `activate` the target toplevel (foreign-toplevel) — focus+raise it, so
-/// on a stacking layout it fills the output.
-/// 2. land a real virtual-pointer button press at the output centre (now over
-/// the target). This both visually clicks and — crucially on sway, whose
-/// headless seat has no keyboard until a pointer interaction occurs — makes
-/// the compositor route the subsequent virtual-keyboard input to the
-/// target. (labwc routes it from `activate` alone; the centre click is
-/// harmless there since the activated window is centred under the cursor.)
-pub fn click(window_id: u64) -> anyhow::Result<()> {
+/// Display-level capture dispatcher. Cascade:
+/// 1. Native Wayland on wlroots: zwlr_screencopy_manager_v1 (fast, zero
+/// consent).
+/// 2. Wayland but no wlroots screencopy globals (GNOME/KDE/COSMIC):
+/// xdg-desktop-portal Screenshot via ashpd. Triggers consent prompt
+/// on first use per session.
+/// 3. X11: existing root-window path.
+pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> {
+ if is_wayland() {
+ // Tier 1: native wlroots screencopy (fast, zero consent).
+ match screenshot_bytes() {
+ Ok(bytes) => return Ok(bytes),
+ Err(e) => {
+ tracing::debug!(
+ "wlroots screencopy unavailable ({e}); trying ext-image-copy-capture-v1"
+ );
+ }
+ }
+ // Tier 2: ext-image-copy-capture-v1 (sway 1.10+, labwc 0.8+, niri,
+ // hyprland, KDE 6.2+, GNOME 47+).
+ match ext_screencopy::screenshot_via_ext_copy() {
+ Ok(bytes) => return Ok(bytes),
+ Err(e) => {
+ tracing::debug!(
+ "ext-image-copy-capture-v1 unavailable ({e}); trying xdg-desktop-portal"
+ );
+ }
+ }
+ // Tier 3: xdg-desktop-portal (GNOME, KDE, COSMIC fallback).
+ match portal_screenshot::screenshot_via_portal() {
+ Ok(bytes) => return Ok(bytes),
+ Err(e) => {
+ tracing::debug!(
+ "xdg-desktop-portal Screenshot unavailable ({e}); falling through to X11"
+ );
+ }
+ }
+ }
+ // Final fallback: X11 root window. Call the X11-only path explicitly
+ // so we don't re-enter screenshot_display_bytes (which routes back here
+ // on Wayland — would loop forever).
+ crate::capture::screenshot_display_bytes_x11()
+}
+
+/// Per-window capture dispatcher. On X11 forwards to the existing window
+/// capture path; on pure Wayland returns a typed error pointing at the
+/// staging `ext-image-copy-capture-v1` protocol — wlr-screencopy is
+/// output-only, and `foreign-toplevel` exposes no per-window geometry to
+/// crop with.
+pub fn screenshot_window_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> {
+ if is_wayland() {
+ anyhow::bail!(
+ "per-window screenshot is not yet supported on native Wayland — \
+ zwlr_screencopy_manager_v1 is output-only and ext-image-copy-capture-v1 \
+ is not yet shipped in wayland-protocols-wlr. Run under XWayland to crop \
+ to a single window, or capture the full output instead."
+ );
+ }
+ crate::capture::screenshot_window_bytes(xid)
+}
+
+// ── Input session helper ─────────────────────────────────────────────────────
+
+/// Live virtual-pointer session: connection + queue + the bound objects every
+/// pointer op (click, scroll, drag) needs. Returned by [`open_vptr_session`].
+pub struct VptrSession {
+ pub conn: Connection,
+ pub queue: wayland_client::EventQueue<State>,
+ pub state: State,
+ pub seat: WlSeat,
+ pub vptr: ZwlrVirtualPointerV1,
+ pub output_w: u32,
+ pub output_h: u32,
+}
+
+/// Bind manager + seat + virtual-pointer + first output, optionally activate a
+/// foreign-toplevel by `window_id` so the synthesised events land on it, and
+/// return the live session that scroll / drag / click reuse. Wayland forbids a
+/// client from knowing another window's on-screen geometry, so we drive every
+/// pointer event in *output* coordinates and rely on the activated toplevel
+/// covering the centre.
+pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<VptrSession> {
let conn = Connection::connect_to_env()?;
let mut queue = conn.new_event_queue::<State>();
let qh = queue.handle();
conn.display().get_registry(&qh, ());
let mut state = State::default();
- queue.roundtrip(&mut state)?; // bind manager + seat + vptr + outputs
+ queue.roundtrip(&mut state)?;
if state.manager.is_none() {
anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1");
}
for _ in 0..4 {
- queue.roundtrip(&mut state)?; // drain toplevel/handle creation + output mode
+ queue.roundtrip(&mut state)?;
}
- let id = window_id as u32;
- let handle = state
- .handles
- .get(&id)
- .ok_or_else(|| anyhow::anyhow!("no native Wayland toplevel for window_id {window_id}"))?
- .clone();
let seat = state
.seat
.clone()
- .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_seat to activate the window"))?;
- handle.activate(&seat);
- queue.roundtrip(&mut state)?; // flush activate so the target is focused/raised
-
- // Land a button press at the output centre (over the now-focused window).
- if let Some(mgr) = state.vptr_manager.clone() {
- let (w, h) = (state.output_w.max(1), state.output_h.max(1));
- let vptr = mgr.create_virtual_pointer(Some(&seat), &qh, ());
- vptr.motion_absolute(0, w / 2, h / 2, w, h);
- vptr.frame();
- vptr.button(0, BTN_LEFT, ButtonState::Pressed);
- vptr.frame();
- vptr.button(0, BTN_LEFT, ButtonState::Released);
- vptr.frame();
- queue.roundtrip(&mut state)?;
- vptr.destroy();
+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_seat for virtual-pointer input"))?;
+ let mgr = state
+ .vptr_manager
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose zwlr_virtual_pointer_manager_v1"))?;
+
+ if let Some(id) = activate_window_id {
+ let handle = state
+ .handles
+ .get(&id)
+ .cloned()
+ .ok_or_else(|| anyhow::anyhow!("no native Wayland toplevel for window_id {id}"))?;
+ handle.activate(&seat);
queue.roundtrip(&mut state)?;
}
+
+ let vptr = mgr.create_virtual_pointer(Some(&seat), &qh, ());
+ let (output_w, output_h) = (state.output_w.max(1), state.output_h.max(1));
+ Ok(VptrSession { conn, queue, state, seat, vptr, output_w, output_h })
+}
+
+/// Map a cua/X11 pointer button (1=left / 2=middle / 3=right) to its evdev
+/// code, which is what `zwlr_virtual_pointer_v1::button` expects.
+pub fn evdev_pointer_button(button: u8) -> u32 {
+ match button {
+ 2 => 0x112, // BTN_MIDDLE
+ 3 => 0x111, // BTN_RIGHT
+ _ => 0x110, // BTN_LEFT
+ }
+}
+
+/// Click a native Wayland toplevel identified by its `window_id` (the
+/// foreign-toplevel protocol id from `list_windows`) at output-relative
+/// `(x, y)`, with `button` (1/2/3 = left/middle/right) emitted `count` times.
+/// Coordinates default to the output centre when both x and y are zero so
+/// the legacy focus-based behaviour is preserved when callers can't supply
+/// real coords. A short delay between iterations gives the compositor time
+/// to discriminate single vs. double clicks.
+pub fn click(window_id: u64, x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> {
+ let mut sess = open_vptr_session(Some(window_id as u32))?;
+ let (w, h) = (sess.output_w, sess.output_h);
+ let (px, py) = if x == 0 && y == 0 {
+ ((w / 2) as i32, (h / 2) as i32)
+ } else {
+ (x, y)
+ };
+ let px = px.clamp(0, w as i32 - 1) as u32;
+ let py = py.clamp(0, h as i32 - 1) as u32;
+ let btn = evdev_pointer_button(button);
+ for i in 0..count.max(1) {
+ if i > 0 {
+ std::thread::sleep(std::time::Duration::from_millis(80));
+ }
+ sess.vptr.motion_absolute(0, px, py, w, h);
+ sess.vptr.frame();
+ sess.vptr.button(0, btn, ButtonState::Pressed);
+ sess.vptr.frame();
+ sess.vptr.button(0, btn, ButtonState::Released);
+ sess.vptr.frame();
+ sess.queue.roundtrip(&mut sess.state)?;
+ }
+ // Keep the synthetic-cursor registry in sync with the warp we just
+ // performed so a subsequent `get_cursor_position` reflects reality.
+ record_synth_cursor(px as i32, py as i32);
+ sess.vptr.destroy();
+ sess.queue.roundtrip(&mut sess.state)?;
+ Ok(())
+}
+
+/// Synthesize a vertical or horizontal scroll on the activated toplevel. Each
+/// tick emits an `axis_source(wheel)` + `axis_discrete(1)` pair through the
+/// virtual-pointer protocol, mirroring how a real wheel notch decomposes. The
+/// magnitude follows wl_pointer convention: ±10 (in wl_fixed = ×256) per tick.
+pub fn scroll(window_id: u64, direction: &str, amount: u32) -> anyhow::Result<()> {
+ let mut sess = open_vptr_session(Some(window_id as u32))?;
+ let (axis, sign): (Axis, i32) = match direction.to_ascii_lowercase().as_str() {
+ "up" => (Axis::VerticalScroll, -1),
+ "down" => (Axis::VerticalScroll, 1),
+ "left" => (Axis::HorizontalScroll, -1),
+ "right" => (Axis::HorizontalScroll, 1),
+ other => anyhow::bail!("unknown scroll direction: {other}"),
+ };
+ // axis_discrete: `value` is logical units (the wayland-rs wrapper
+ // converts to wl_fixed internally); `discrete` is the tick count.
+ let value: f64 = (sign as f64) * 10.0;
+ for i in 0..amount.max(1) {
+ if i > 0 {
+ std::thread::sleep(std::time::Duration::from_millis(25));
+ }
+ sess.vptr.axis_source(AxisSource::Wheel);
+ sess.vptr.axis_discrete(0, axis, value, sign);
+ sess.vptr.frame();
+ sess.queue.roundtrip(&mut sess.state)?;
+ }
+ sess.vptr.destroy();
+ sess.queue.roundtrip(&mut sess.state)?;
+ Ok(())
+}
+
+/// Last cursor position the agent warped to via `move_cursor_absolute`.
+/// Wayland exposes no protocol for clients to read the real global cursor
+/// position; a Wayland-conformant `get_cursor_position` can therefore only
+/// report what THIS process synthesized. Updated every `motion_absolute`
+/// emitted from `move_cursor_absolute` / `click` / `drag`.
+static SYNTH_CURSOR_POS: std::sync::OnceLock<std::sync::Mutex<Option<(i32, i32)>>> =
+ std::sync::OnceLock::new();
+
+fn record_synth_cursor(x: i32, y: i32) {
+ let cell = SYNTH_CURSOR_POS.get_or_init(|| std::sync::Mutex::new(None));
+ if let Ok(mut g) = cell.lock() {
+ *g = Some((x, y));
+ }
+}
+
+/// Returns the last `(x, y)` this process warped the cursor to via the
+/// Wayland virtual-pointer protocol, or `None` if no warp has happened in
+/// this process. The reading is "synthetic": Wayland forbids clients from
+/// querying the real cursor position, so this value diverges from reality
+/// the moment the user moves their physical mouse. Callers should surface
+/// `source: "synthetic"` in the structured payload.
+pub fn last_synth_cursor_pos() -> Option<(i32, i32)> {
+ SYNTH_CURSOR_POS.get_or_init(|| std::sync::Mutex::new(None))
+ .lock()
+ .ok()
+ .and_then(|g| *g)
+}
+
+/// Warp the cursor to absolute output coordinates `(x, y)` using
+/// `zwlr_virtual_pointer_v1::motion_absolute`. Clamps to the output bounds
+/// reported by `open_vptr_session`. Emits a motion + frame and roundtrips so
+/// the compositor commits the warp before returning. Records the position in
+/// the synthetic-cursor registry so `last_synth_cursor_pos` can report it.
+pub fn move_cursor_absolute(window_id: Option<u64>, x: i32, y: i32) -> anyhow::Result<()> {
+ let mut sess = open_vptr_session(window_id.map(|w| w as u32))?;
+ let (w, h) = (sess.output_w, sess.output_h);
+ let px = x.clamp(0, (w as i32).saturating_sub(1)) as u32;
+ let py = y.clamp(0, (h as i32).saturating_sub(1)) as u32;
+ sess.vptr.motion_absolute(0, px, py, w, h);
+ sess.vptr.frame();
+ sess.queue.roundtrip(&mut sess.state)?;
+ record_synth_cursor(px as i32, py as i32);
+ sess.vptr.destroy();
+ sess.queue.roundtrip(&mut sess.state)?;
+ Ok(())
+}
+
+/// Press-drag-release on a native Wayland toplevel. Emits one button press at
+/// `(from_x, from_y)`, then `steps` interpolated motion events along the
+/// straight segment to `(to_x, to_y)`, then a release. Coordinates are
+/// output-relative; window-local coords need the EIS inject socket
+/// (`CUA_INJECT_SOCKET`).
+pub fn drag(
+ window_id: u64,
+ from_x: i32,
+ from_y: i32,
+ to_x: i32,
+ to_y: i32,
+ steps: u32,
+ button: u8,
+) -> anyhow::Result<()> {
+ let mut sess = open_vptr_session(Some(window_id as u32))?;
+ let (w, h) = (sess.output_w, sess.output_h);
+ let btn = evdev_pointer_button(button);
+ let clamp_xy = |x: i32, y: i32| -> (u32, u32) {
+ (
+ x.clamp(0, w as i32 - 1) as u32,
+ y.clamp(0, h as i32 - 1) as u32,
+ )
+ };
+ let (fx, fy) = clamp_xy(from_x, from_y);
+ sess.vptr.motion_absolute(0, fx, fy, w, h);
+ sess.vptr.frame();
+ sess.vptr.button(0, btn, ButtonState::Pressed);
+ sess.vptr.frame();
+ sess.queue.roundtrip(&mut sess.state)?;
+ let n = steps.max(1);
+ for s in 1..=n {
+ let t = s as f64 / n as f64;
+ let ix = (from_x as f64 + (to_x - from_x) as f64 * t).round() as i32;
+ let iy = (from_y as f64 + (to_y - from_y) as f64 * t).round() as i32;
+ let (cx, cy) = clamp_xy(ix, iy);
+ sess.vptr.motion_absolute(0, cx, cy, w, h);
+ sess.vptr.frame();
+ sess.queue.roundtrip(&mut sess.state)?;
+ std::thread::sleep(std::time::Duration::from_millis(8));
+ }
+ let (tx, ty) = clamp_xy(to_x, to_y);
+ sess.vptr.motion_absolute(0, tx, ty, w, h);
+ sess.vptr.frame();
+ sess.vptr.button(0, btn, ButtonState::Released);
+ sess.vptr.frame();
+ // Sync the synthetic-cursor registry with the drag endpoint so a
+ // subsequent `get_cursor_position` reports where we left the pointer.
+ record_synth_cursor(tx as i32, ty as i32);
+ sess.queue.roundtrip(&mut sess.state)?;
+ sess.vptr.destroy();
+ sess.queue.roundtrip(&mut sess.state)?;
Ok(())
}
@@ -424,6 +1012,59 @@ pub fn press_key(key: &str) -> anyhow::Result<()> {
Ok(())
}
+/// Press a key combination (modifiers + final key) via `wtype`. Each modifier
+/// is pressed before the key, then released after, exactly like
+/// `wtype -M ctrl -M shift -k key -m shift -m ctrl`. Unknown values pass
+/// straight to wtype's `-k` so single-character keys and X keysym names work
+/// as-is. This is the Wayland equivalent of the X11 `send_key` modifier mask.
+pub fn hotkey(keys: &[String]) -> anyhow::Result<()> {
+ let (mods, final_key) = partition_modifiers(keys)?;
+ let keysym = key_to_keysym(&final_key);
+ let mut args: Vec<String> = Vec::new();
+ for m in &mods {
+ args.push("-M".into());
+ args.push(m.clone());
+ }
+ args.push("-k".into());
+ args.push(keysym.clone());
+ // Release modifiers in reverse press order.
+ for m in mods.iter().rev() {
+ args.push("-m".into());
+ args.push(m.clone());
+ }
+ let out = std::process::Command::new("wtype").args(&args).output()?;
+ if !out.status.success() {
+ anyhow::bail!(
+ "wtype {} failed: {}",
+ args.join(" "),
+ String::from_utf8_lossy(&out.stderr)
+ );
+ }
+ Ok(())
+}
+
+/// Split a `keys` array into wtype-compatible modifier names and a single
+/// final key. Recognised modifier inputs: ctrl/control, alt, shift,
+/// super/meta/cmd/command/win/windows. The final key must be the one
+/// non-modifier in the list.
+fn partition_modifiers(keys: &[String]) -> anyhow::Result<(Vec<String>, String)> {
+ let mut mods: Vec<String> = Vec::new();
+ let mut non_mods: Vec<String> = Vec::new();
+ for k in keys {
+ match k.to_ascii_lowercase().as_str() {
+ "ctrl" | "control" => mods.push("ctrl".into()),
+ "alt" => mods.push("alt".into()),
+ "shift" => mods.push("shift".into()),
+ "super" | "meta" | "cmd" | "command" | "win" | "windows" => mods.push("logo".into()),
+ _ => non_mods.push(k.clone()),
+ }
+ }
+ let final_key = non_mods.last().cloned().ok_or_else(|| {
+ anyhow::anyhow!("hotkey requires at least one non-modifier key")
+ })?;
+ Ok((mods, final_key))
+}
+
/// Map cua key names to X keysym names that `wtype -k` understands. Unknown
/// values pass through (single characters and valid keysym names work as-is).
fn key_to_keysym(key: &str) -> String {
@@ -510,6 +1151,20 @@ pub fn app_id_for_window(window_id: u64) -> Option<String> {
.filter(|s| !s.is_empty())
}
+/// Resolve the WM_CLASS-equivalent (instance, class) pair for a window. On
+/// X11 reads `WM_CLASS`; on Wayland reuses [`app_id_for_window`] and returns
+/// `(app_id, app_id)` — the closest analogue, since foreign-toplevel exposes
+/// a single app_id and not the X11 instance/class split. Used by terminal
+/// emulator detection on `is_terminal_window` so Ghostty / kitty / alacritty
+/// are recognised on Wayland too.
+pub fn wm_class_dispatch(window_id: u64) -> Option<(String, String)> {
+ if is_wayland() {
+ let app = app_id_for_window(window_id)?;
+ return Some((app.clone(), app));
+ }
+ crate::x11::wm_class_for_window(window_id)
+}
+
fn to_hex(s: &str) -> String {
s.bytes().map(|b| format!("{b:02x}")).collect()
}
@@ -619,3 +1274,97 @@ pub fn list_windows_dispatch(filter_pid: Option<u32>) -> Vec<WindowInfo> {
}
crate::x11::list_windows(filter_pid)
}
+
+/// Snapshot of which wlroots manager globals the running compositor advertises.
+/// Used by the `health_report` Wayland backend probe to distinguish a working
+/// session from one missing screencopy or virtual-pointer support.
+#[derive(Default, Clone, Debug)]
+pub struct WaylandManagers {
+ pub foreign_toplevel: bool,
+ pub screencopy: bool,
+ pub virtual_pointer: bool,
+ pub wl_shm: bool,
+ /// Staging `ext-image-copy-capture-v1` manager — sway 1.10+, labwc
+ /// 0.8+, niri, KDE 6.2+, GNOME mutter 47+.
+ pub ext_image_copy_capture: bool,
+ /// Companion `ext-output-image-capture-source-v1` source manager —
+ /// required to capture a `wl_output` via the staging protocol.
+ pub ext_output_image_capture_source: bool,
+}
+
+/// Perform a single registry roundtrip and report which of the manager
+/// interfaces the doctor cares about advertise themselves. Returns `Err` only
+/// when we can't even open a Wayland connection — a successful connect with
+/// no managers still resolves to an all-false snapshot.
+pub fn probe_managers() -> anyhow::Result<WaylandManagers> {
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<State>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+ let mut state = State::default();
+ queue.roundtrip(&mut state)?;
+ // Reuse the existing State for wlroots managers, then do a parallel
+ // probe for staging ext-image-copy-capture interfaces by walking the
+ // raw registry events (no binding required — we only need presence).
+ let ext_probe = probe_ext_interfaces().unwrap_or_default();
+ Ok(WaylandManagers {
+ foreign_toplevel: state.manager.is_some(),
+ screencopy: state.scrcopy_manager.is_some(),
+ virtual_pointer: state.vptr_manager.is_some(),
+ wl_shm: state.shm.is_some(),
+ ext_image_copy_capture: ext_probe.image_copy_capture,
+ ext_output_image_capture_source: ext_probe.output_image_capture_source,
+ })
+}
+
+#[derive(Default, Clone, Copy, Debug)]
+struct ExtInterfaceProbe {
+ image_copy_capture: bool,
+ output_image_capture_source: bool,
+}
+
+/// Probe registry for `ext-image-copy-capture-v1` + companion source manager
+/// presence without binding them. Cheap (one roundtrip) and side-effect
+/// free.
+fn probe_ext_interfaces() -> anyhow::Result<ExtInterfaceProbe> {
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<ExtProbeState>();
+ let qh = queue.handle();
+ conn.display().get_registry(&qh, ());
+ let mut state = ExtProbeState::default();
+ queue.roundtrip(&mut state)?;
+ Ok(state.probe)
+}
+
+#[derive(Default)]
+struct ExtProbeState {
+ probe: ExtInterfaceProbe,
+}
+
+impl Dispatch<wl_registry::WlRegistry, ()> for ExtProbeState {
+ fn event(
+ state: &mut Self,
+ _: &wl_registry::WlRegistry,
+ event: wl_registry::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ if let wl_registry::Event::Global { interface, .. } = event {
+ match interface.as_str() {
+ "ext_image_copy_capture_manager_v1" => {
+ state.probe.image_copy_capture = true;
+ }
+ "ext_output_image_capture_source_manager_v1" => {
+ state.probe.output_image_capture_source = true;
+ }
+ _ => {}
+ }
+ }
+ }
+}
+
+// Suppress dead-code warning for the unused BTN_LEFT alias kept for backward
+// compatibility with earlier slice constants.
+#[allow(dead_code)]
+const _BTN_LEFT_ALIAS: u32 = BTN_LEFT;
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs
new file mode 100644
index 0000000000..7ffac0a279
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/overlay.rs
@@ -0,0 +1,609 @@
+//! Native-Wayland agent-cursor overlay via `zwlr_layer_shell_v1`.
+//!
+//! Replaces the X11-only `overlay.rs` render loop on wlroots compositors
+//! (sway, labwc, kwin 5.27+, hyprland) by creating a full-screen,
+//! click-through, always-on-top `wl_surface` anchored to the first output
+//! via `zwlr_layer_shell_v1`. The surface renders the same gradient-arrow
+//! cursor as the X11 path by sharing `cursor_overlay::RenderStateCore` —
+//! bloom, click-pulse, idle-fade, and motion all work identically.
+//!
+//! GNOME mutter does not expose `zwlr_layer_shell_v1` — those sessions
+//! either fall through to the X11 path (XWayland) or the nested-compositor
+//! mode that spawns labwc internally.
+//!
+//! Architecture mirrors the existing `wayland/persistent_vptr.rs`: one
+//! owner thread (`cua-overlay-wl`) holds the wayland Connection +
+//! EventQueue + layer surface; commands flow in over a `crossbeam-channel`.
+//! The render core is ticked at ~60Hz via a calloop timer so motion +
+//! spring physics + click pulse advance smoothly even when no new
+//! Position command has arrived.
+
+use std::collections::HashMap;
+use std::sync::OnceLock;
+use std::thread;
+use std::time::Instant;
+
+use crossbeam_channel::{bounded, Receiver, Sender};
+use cursor_overlay::{CursorConfig, OverlayCommand, OverlayMsg, CursorKey, RenderStateCore};
+use wayland_client::{
+ protocol::{
+ wl_buffer::WlBuffer,
+ wl_compositor::WlCompositor,
+ wl_output::WlOutput,
+ wl_region::WlRegion,
+ wl_registry,
+ wl_shm::{self, WlShm},
+ wl_shm_pool::WlShmPool,
+ wl_surface::WlSurface,
+ },
+ Connection, Dispatch, Proxy, QueueHandle,
+};
+use wayland_protocols_wlr::layer_shell::v1::client::{
+ zwlr_layer_shell_v1::{self, Layer, ZwlrLayerShellV1},
+ zwlr_layer_surface_v1::{self, Anchor, KeyboardInteractivity, ZwlrLayerSurfaceV1},
+};
+
+/// Commands the overlay owner thread accepts. The richer commands the
+/// cross-platform [`RenderStateCore`] understands (MoveTo, ClickPulse,
+/// SetPressed) are forwarded as-is so the layer-shell overlay matches the
+/// X11 visual: bloom + animated arrow + click pulse + press ring.
+enum WlOverlayCmd {
+ Cmd { key: CursorKey, cmd: OverlayCommand },
+ Remove { key: CursorKey },
+ Shutdown,
+}
+
+static TX: OnceLock<Sender<WlOverlayCmd>> = OnceLock::new();
+
+fn tx() -> Option<&'static Sender<WlOverlayCmd>> {
+ TX.get()
+}
+
+/// Lazily start the owner thread. Idempotent — safe to call from every
+/// MCP tool invocation; subsequent calls are no-ops.
+pub fn ensure_started() {
+ TX.get_or_init(|| {
+ let (tx, rx) = bounded::<WlOverlayCmd>(64);
+ thread::Builder::new()
+ .name("cua-overlay-wl".into())
+ .spawn(move || {
+ if let Err(e) = owner_thread(rx) {
+ tracing::warn!("cua-overlay-wl thread exited with error: {e}");
+ }
+ })
+ .expect("spawn cua-overlay-wl thread");
+ tx
+ });
+}
+
+/// Translate a generic [`OverlayMsg`] (the cross-platform command shape)
+/// to the layer-shell owner thread. The owner-thread render core consumes
+/// every variant the X11 path handles; only `ShowFocusRect` (macOS-only)
+/// is silently dropped here.
+pub fn forward(msg: &OverlayMsg) -> bool {
+ // Lazy startup: spawning the layer-shell owner thread + connecting to
+ // the Wayland compositor takes 100-300ms. Doing that at cua-driver mcp
+ // boot (the old eager-init path) was tipping the borderline CI
+ // cursor-click-gif test over its 20s budget. ensure_started is
+ // idempotent so calling it on every forward is fine — the OnceLock
+ // bypasses the spawn after the first call.
+ ensure_started();
+ let Some(tx) = tx() else { return false };
+ match msg {
+ OverlayMsg::Remove(k) => {
+ let _ = tx.try_send(WlOverlayCmd::Remove { key: k.clone() });
+ true
+ }
+ OverlayMsg::Cmd(kc) => {
+ if matches!(&kc.cmd, OverlayCommand::ShowFocusRect(_)) {
+ return false;
+ }
+ let _ = tx.try_send(WlOverlayCmd::Cmd {
+ key: kc.key.clone(),
+ cmd: kc.cmd.clone(),
+ });
+ true
+ }
+ }
+}
+
+/// Cleanly stop the owner thread. Tests use this; production code typically
+/// lets the thread die at process exit.
+pub fn shutdown() {
+ if let Some(tx) = tx() {
+ let _ = tx.send(WlOverlayCmd::Shutdown);
+ }
+}
+
+// ── owner thread ─────────────────────────────────────────────────────────
+
+struct OverlayState {
+ compositor: Option<WlCompositor>,
+ shm: Option<WlShm>,
+ layer_shell: Option<ZwlrLayerShellV1>,
+ output: Option<WlOutput>,
+ output_w: u32,
+ output_h: u32,
+ surface: Option<WlSurface>,
+ layer_surface: Option<ZwlrLayerSurfaceV1>,
+ configured: bool,
+ /// Cross-platform render core: position, animation, gradient arrow,
+ /// bloom, click pulse, idle-fade. Shared verbatim with the X11 path.
+ core: RenderStateCore,
+ /// In-flight wl_shm buffers awaiting `wl_buffer.release` from the
+ /// compositor. Keyed by `WlBuffer` object id; value is the
+ /// `(mmap ptr, mmap size, memfd fd)` triple that must be unmapped +
+ /// closed once the compositor signals it's done with the buffer.
+ /// Replaces the per-redraw `mem::forget` leak: the previous frame's
+ /// memory is reclaimed as soon as the compositor releases it.
+ pending_buffers: HashMap<u32, (*mut libc::c_void, usize, i32)>,
+}
+
+// SAFETY: the raw pointers in pending_buffers point at mmap regions owned
+// exclusively by this thread (the owner thread). OverlayState is never
+// shared across threads — wayland-client's EventQueue<State> is !Send so
+// it stays pinned to the owner thread. The Send/Sync bounds wayland-client
+// requires for State types apply to the struct as a whole, hence the
+// explicit assertion.
+unsafe impl Send for OverlayState {}
+
+impl Default for OverlayState {
+ fn default() -> Self {
+ Self {
+ compositor: None,
+ shm: None,
+ layer_shell: None,
+ output: None,
+ output_w: 0,
+ output_h: 0,
+ surface: None,
+ layer_surface: None,
+ configured: false,
+ core: RenderStateCore::new(CursorConfig::default()),
+ pending_buffers: HashMap::new(),
+ }
+ }
+}
+
+fn dbg(msg: &str) {
+ if std::env::var_os("CUA_OVERLAY_DEBUG").is_some() {
+ eprintln!("[cua-overlay-wl] {msg}");
+ }
+}
+
+fn owner_thread(rx: Receiver<WlOverlayCmd>) -> anyhow::Result<()> {
+ let conn = Connection::connect_to_env()?;
+ let mut queue = conn.new_event_queue::<OverlayState>();
+ let qh = queue.handle();
+ let _registry = conn.display().get_registry(&qh, ());
+
+ let mut state = OverlayState::default();
+ queue.roundtrip(&mut state)?;
+ for _ in 0..3 {
+ queue.roundtrip(&mut state)?;
+ }
+
+ let compositor = state
+ .compositor
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_compositor"))?;
+ let shm = state
+ .shm
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose wl_shm"))?;
+ let layer_shell = state
+ .layer_shell
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor does not expose zwlr_layer_shell_v1"))?;
+ let output = state
+ .output
+ .clone()
+ .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_output"))?;
+
+ // Build the layer surface: fullscreen, overlay layer, click-through.
+ let surface = compositor.create_surface(&qh, ());
+ let layer_surface = layer_shell.get_layer_surface(
+ &surface,
+ Some(&output),
+ Layer::Overlay,
+ "cua-agent-cursor".to_string(),
+ &qh,
+ (),
+ );
+ // Anchor to all four edges = full screen.
+ layer_surface.set_anchor(Anchor::Top | Anchor::Bottom | Anchor::Left | Anchor::Right);
+ layer_surface.set_size(0, 0);
+ layer_surface.set_exclusive_zone(-1);
+ layer_surface.set_keyboard_interactivity(KeyboardInteractivity::None);
+
+ // Click-through: empty input region.
+ let region: WlRegion = compositor.create_region(&qh, ());
+ surface.set_input_region(Some(&region));
+ region.destroy();
+
+ state.surface = Some(surface);
+ state.layer_surface = Some(layer_surface);
+
+ // First commit kicks off the configure handshake.
+ if let Some(s) = state.surface.as_ref() {
+ s.commit();
+ }
+
+ // Wait for the first configure event so we know the output dimensions
+ // before drawing.
+ for _ in 0..10 {
+ queue.roundtrip(&mut state)?;
+ if state.configured && state.output_w > 0 && state.output_h > 0 {
+ break;
+ }
+ std::thread::sleep(std::time::Duration::from_millis(50));
+ }
+
+ if !state.configured {
+ anyhow::bail!("layer surface never received configure event");
+ }
+ dbg(&format!("configured: w={} h={}", state.output_w, state.output_h));
+
+ // Main loop. Tick the render core at ~60Hz so motion + spring physics
+ // + click pulse animate smoothly; redraw every tick when the cursor is
+ // visible. Commands arriving via the channel update the render core
+ // first, then the next tick paints the result.
+ redraw(&mut state, &shm, &qh)?;
+ queue.roundtrip(&mut state)?;
+
+ let frame_dur = std::time::Duration::from_millis(16);
+ let mut last_tick = Instant::now();
+ loop {
+ // Drain all pending commands without blocking.
+ let mut shutdown = false;
+ loop {
+ match rx.try_recv() {
+ Ok(WlOverlayCmd::Shutdown) => { shutdown = true; break; }
+ Ok(WlOverlayCmd::Cmd { cmd, .. }) => {
+ // Seed: if the cursor is still at the off-screen sentinel
+ // `(-200, -200)` from `RenderStateCore::new`, snap to a
+ // point near the MoveTo / SnapTo target so the spring
+ // animation starts on-screen. Mirrors X11 overlay.rs's
+ // `seed_start_if_sentinel` helper — without it, the
+ // spring oscillates around the sentinel and the cursor
+ // never reaches the screen.
+ let seed_target = match &cmd {
+ OverlayCommand::MoveTo { x, y, .. }
+ | OverlayCommand::SnapTo { x, y, .. }
+ | OverlayCommand::ClickPulse { x, y } => Some((*x, *y)),
+ _ => None,
+ };
+ if let Some((tx, ty)) = seed_target {
+ if state.core.pos.0 < -50.0 {
+ const SEED_OFFSET: f64 = 16.0;
+ let sx = (tx - SEED_OFFSET).max(2.0);
+ let sy = (ty - SEED_OFFSET).max(2.0);
+ state.core.pos = (sx, sy);
+ }
+ }
+ // apply_command_base consumes every variant the X11
+ // path handles. `move_to_snap_sentinel` / `click_pulse
+ // _sentinel_only` are both `false` here — same as X11.
+ let _ = state.core.apply_command_base(cmd, false, false);
+ }
+ Ok(WlOverlayCmd::Remove { .. }) => {
+ // Single-cursor overlay: removing the active cursor
+ // hides it. Multi-cursor wlroots support can layer on
+ // top of this in a follow-up if needed.
+ state.core.visible = false;
+ }
+ Err(crossbeam_channel::TryRecvError::Empty) => break,
+ Err(crossbeam_channel::TryRecvError::Disconnected) => { shutdown = true; break; }
+ }
+ }
+ if shutdown { break; }
+
+ // Tick animation forward and repaint if anything changed.
+ let now = Instant::now();
+ let dt = now.duration_since(last_tick).as_secs_f64().min(0.05);
+ last_tick = now;
+ state.core.tick_motion(dt);
+ if state.configured {
+ redraw(&mut state, &shm, &qh)?;
+ }
+ queue.dispatch_pending(&mut state)?;
+
+ // Sleep for the remainder of the frame budget so the loop doesn't
+ // spin. Channel-driven wakeups would be lower-latency, but layer
+ // overlays only need to keep up with display refresh.
+ let elapsed = last_tick.elapsed();
+ if elapsed < frame_dur {
+ std::thread::sleep(frame_dur - elapsed);
+ }
+ }
+
+ if let Some(ls) = state.layer_surface.take() {
+ ls.destroy();
+ }
+ if let Some(s) = state.surface.take() {
+ s.destroy();
+ }
+ queue.roundtrip(&mut state)?;
+ Ok(())
+}
+
+/// Render one cursor frame into a fresh wl_shm ARGB8888 buffer and attach
+/// it to the layer surface.
+///
+/// Pipeline:
+/// 1. Allocate a memfd-backed wl_shm pool sized at output_w × output_h.
+/// 2. Paint the cross-platform cursor (bloom + click pulse + gradient
+/// arrow) into a `tiny_skia::Pixmap` via `cursor_overlay::paint_cursor`
+/// — same call the X11 path uses.
+/// 3. Channel-swap RGBA → BGRA into the wl_shm buffer (wl_shm Argb8888
+/// is little-endian BGRA in memory). This is the inverse of the swap
+/// in `ext_screencopy::encode_buffer_to_png`.
+/// 4. Attach + damage + commit on the layer surface.
+///
+/// When the cursor is hidden (`core.visible == false`, idle-faded, or
+/// off-screen sentinel) the pixmap is all zeros — the surface remains
+/// transparent and click-through.
+fn redraw(state: &mut OverlayState, shm: &WlShm, qh: &QueueHandle<OverlayState>) -> anyhow::Result<()> {
+ let Some(surface) = state.surface.as_ref() else { return Ok(()) };
+ let w = state.output_w.max(1);
+ let h = state.output_h.max(1);
+ let stride = w as i32 * 4;
+ let size = (stride as usize) * (h as usize);
+
+ // Reuses the same anon_shm pattern as the screencopy path in mod.rs.
+ let (fd, ptr) = super::anon_shm(size)
+ .map_err(|e| anyhow::anyhow!("overlay shm allocation failed: {e}"))?;
+
+ // SAFETY: ptr came from mmap of `size` bytes, lifetime bounded to this
+ // function.
+ let pixels: &mut [u8] = unsafe { std::slice::from_raw_parts_mut(ptr as *mut u8, size) };
+
+ // Paint the cursor into a tiny_skia pixmap. paint_cursor early-returns
+ // when the cursor is hidden / off-screen / idle-faded, so the pixmap
+ // is left fully transparent in those cases (which is also what we want
+ // for the click-through layer surface).
+ let pm_result = tiny_skia::Pixmap::new(w, h);
+ let mut pm = match pm_result {
+ Some(p) => p,
+ // tiny_skia::Pixmap::new only fails on OOM at sizes that fit u32.
+ // We refuse to fall back to a 1x1 pixmap because the subsequent
+ // RGBA → BGRA loop indexes `src[i+3]` over the full `size` range —
+ // a 1x1 source would crash. Surface the allocation failure
+ // properly instead.
+ None => anyhow::bail!(
+ "tiny_skia::Pixmap::new({w}, {h}) failed — out of memory for the overlay buffer"
+ ),
+ };
+ // backing_scale=1.0 matches the X11 path; per-output Wayland scale is
+ // a follow-up (would consume `wl_output.scale` and `preferred_buffer
+ // _scale` from wl_surface v6).
+ cursor_overlay::paint_cursor(&mut pm, &state.core, 0.0, 0.0, None, 1.0);
+
+ // CUA_OVERLAY_DEBUG=1 paints a 60x60 magenta square at the cursor's
+ // current pos on top of the gradient arrow. Useful when validating
+ // layer-shell visibility on a new compositor — the gradient arrow is
+ // small at native scale and easy to miss in a screenshot, while the
+ // magenta block is impossible to miss.
+ if std::env::var_os("CUA_OVERLAY_DEBUG").is_some() {
+ let (cx, cy) = state.core.pos;
+ let cx = cx as i32;
+ let cy = cy as i32;
+ let half = 30i32;
+ for dy in -half..half {
+ for dx in -half..half {
+ let px = cx + dx;
+ let py = cy + dy;
+ if px < 0 || py < 0 || px >= w as i32 || py >= h as i32 {
+ continue;
+ }
+ let off = ((py as usize) * (w as usize) + (px as usize)) * 4;
+ pm.data_mut()[off] = 0xFF; // R
+ pm.data_mut()[off + 1] = 0x00; // G
+ pm.data_mut()[off + 2] = 0xFF; // B
+ pm.data_mut()[off + 3] = 0xFF; // A
+ }
+ }
+ }
+
+// RGBA → BGRA channel swap. tiny_skia stores pixels as RGBA8888
+ // (premultiplied); wl_shm Argb8888 is little-endian = BGRA in memory.
+ // Mirrors the inverse swap in ext_screencopy::encode_buffer_to_png.
+ let src = pm.data();
+ for i in (0..size).step_by(4) {
+ // pm.data() is already RGBA premultiplied; just swap R↔B.
+ pixels[i] = src[i + 2]; // B ← R
+ pixels[i + 1] = src[i + 1]; // G
+ pixels[i + 2] = src[i]; // R ← B
+ pixels[i + 3] = src[i + 3]; // A
+ }
+
+ use std::os::fd::AsFd as _;
+ let pool_fd = unsafe { super::borrowed_fd(fd) };
+ let pool: WlShmPool = shm.create_pool(pool_fd.as_fd(), size as i32, qh, ());
+ let buffer: WlBuffer = pool.create_buffer(
+ 0,
+ w as i32,
+ h as i32,
+ stride,
+ wl_shm::Format::Argb8888,
+ qh,
+ (),
+ );
+
+ // Track the (mmap, fd) by buffer object id so the wl_buffer.release
+ // event Dispatch handler can clean up exactly when the compositor is
+ // done with the underlying memory — no leak, no use-after-free.
+ let buffer_id = buffer.id().protocol_id();
+ state.pending_buffers.insert(buffer_id, (ptr, size, fd));
+
+ dbg(&format!("redraw w={w} h={h} stride={stride} buf_id={buffer_id} pos=({:.1},{:.1}) visible={}", state.core.pos.0, state.core.pos.1, state.core.visible));
+ surface.attach(Some(&buffer), 0, 0);
+ surface.damage_buffer(0, 0, w as i32, h as i32);
+ surface.commit();
+ pool.destroy();
+ Ok(())
+}
+
+// ── Wayland Dispatch impls ───────────────────────────────────────────────
+
+impl Dispatch<wl_registry::WlRegistry, ()> for OverlayState {
+ fn event(
+ state: &mut Self,
+ registry: &wl_registry::WlRegistry,
+ event: wl_registry::Event,
+ _data: &(),
+ _conn: &Connection,
+ qh: &QueueHandle<Self>,
+ ) {
+ if let wl_registry::Event::Global { name, interface, version } = event {
+ match interface.as_str() {
+ "wl_compositor" => {
+ state.compositor = Some(registry.bind::<WlCompositor, _, _>(
+ name, version.min(6), qh, ()));
+ }
+ "wl_shm" => {
+ state.shm = Some(registry.bind::<WlShm, _, _>(name, version.min(1), qh, ()));
+ }
+ "wl_output" => {
+ if state.output.is_none() {
+ state.output = Some(registry.bind::<WlOutput, _, _>(name, version.min(4), qh, ()));
+ }
+ }
+ "zwlr_layer_shell_v1" => {
+ state.layer_shell = Some(registry.bind::<ZwlrLayerShellV1, _, _>(
+ name, version.min(4), qh, ()));
+ }
+ _ => {}
+ }
+ }
+ }
+}
+
+impl Dispatch<WlCompositor, ()> for OverlayState {
+ fn event(
+ _state: &mut Self,
+ _: &WlCompositor,
+ _: <WlCompositor as wayland_client::Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {}
+}
+
+impl Dispatch<WlShm, ()> for OverlayState {
+ fn event(
+ _state: &mut Self,
+ _: &WlShm,
+ _: <WlShm as wayland_client::Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {}
+}
+
+impl Dispatch<WlOutput, ()> for OverlayState {
+ fn event(
+ state: &mut Self,
+ _: &WlOutput,
+ event: <WlOutput as wayland_client::Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ use wayland_client::protocol::wl_output;
+ if let wl_output::Event::Mode { width, height, .. } = event {
+ if width > 0 && height > 0 {
+ state.output_w = width as u32;
+ state.output_h = height as u32;
+ }
+ }
+ }
+}
+
+impl Dispatch<ZwlrLayerShellV1, ()> for OverlayState {
+ fn event(
+ _state: &mut Self,
+ _: &ZwlrLayerShellV1,
+ _: <ZwlrLayerShellV1 as wayland_client::Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {}
+}
+
+impl Dispatch<ZwlrLayerSurfaceV1, ()> for OverlayState {
+ fn event(
+ state: &mut Self,
+ layer: &ZwlrLayerSurfaceV1,
+ event: <ZwlrLayerSurfaceV1 as wayland_client::Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ if let zwlr_layer_surface_v1::Event::Configure { serial, width, height } = event {
+ layer.ack_configure(serial);
+ if width > 0 {
+ state.output_w = width;
+ }
+ if height > 0 {
+ state.output_h = height;
+ }
+ state.configured = true;
+ }
+ }
+}
+
+impl Dispatch<WlSurface, ()> for OverlayState {
+ fn event(
+ _: &mut Self,
+ _: &WlSurface,
+ _: <WlSurface as wayland_client::Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {}
+}
+
+impl Dispatch<WlShmPool, ()> for OverlayState {
+ fn event(
+ _: &mut Self,
+ _: &WlShmPool,
+ _: <WlShmPool as wayland_client::Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {}
+}
+
+impl Dispatch<WlBuffer, ()> for OverlayState {
+ fn event(
+ state: &mut Self,
+ buffer: &WlBuffer,
+ event: <WlBuffer as wayland_client::Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {
+ use wayland_client::protocol::wl_buffer;
+ if matches!(event, wl_buffer::Event::Release) {
+ // Compositor is done with the underlying mmap. Free it +
+ // close the memfd + destroy the wayland object.
+ let id = buffer.id().protocol_id();
+ if let Some((ptr, size, fd)) = state.pending_buffers.remove(&id) {
+ super::cleanup_mmap(ptr, size, fd);
+ }
+ buffer.destroy();
+ }
+ }
+}
+
+impl Dispatch<WlRegion, ()> for OverlayState {
+ fn event(
+ _: &mut Self,
+ _: &WlRegion,
+ _: <WlRegion as wayland_client::Proxy>::Event,
+ _: &(),
+ _: &Connection,
+ _: &QueueHandle<Self>,
+ ) {}
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs
new file mode 100644
index 0000000000..7f3eaa2bcd
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs
@@ -0,0 +1,287 @@
+//! Persistent virtual-pointer for stateful `mouse_button_down` / `mouse_drag` /
+//! `mouse_button_up`.
+//!
+//! The non-persistent path in [`crate::wayland::click`] opens its own
+//! `ZwlrVirtualPointerV1`, presses, releases, drops the connection — useful
+//! for one-shot clicks but useless for held-button drags: each tool call
+//! emits a fresh device whose press/release pair is matched by the
+//! compositor, so apps that distinguish a real drag (press, motion+, release)
+//! from a series of clicks (press, release, press, release, …) miss the
+//! drag entirely.
+//!
+//! This module keeps the virtual-pointer alive across tool calls. A single
+//! owner thread per process owns one Wayland `Connection`, one `EventQueue`,
+//! and a map of `cursor_id -> ActivePointer`. Commands are sent over a
+//! `crossbeam-channel`; replies come back on a per-call reply channel so the
+//! caller blocks until the compositor has roundtripped.
+//!
+//! Lifecycle:
+//! - First `press` for a cursor_id binds a fresh `ZwlrVirtualPointerV1`,
+//! activates the foreign-toplevel target window once, presses the button,
+//! adds to the held-button set, roundtrips.
+//! - Subsequent `move_to` calls emit `motion_absolute` on the same vptr (no
+//! activate — would steal focus mid-drag) and roundtrip.
+//! - `release` emits a button release, removes from the held set; if the
+//! set is empty the vptr is destroyed and the map entry dropped.
+//! - On `Connection` roundtrip failure (compositor restart / disconnect)
+//! the owner thread tears down its connection and accepts the next
+//! command on a fresh one, emitting a typed error for the in-flight call.
+
+use std::collections::{HashMap, HashSet};
+use std::sync::OnceLock;
+use std::thread;
+
+use crossbeam_channel::{bounded, Receiver, Sender};
+use wayland_client::{
+ protocol::wl_pointer::ButtonState,
+ Connection, Proxy,
+};
+use wayland_protocols_wlr::virtual_pointer::v1::client::zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1;
+
+use super::{open_vptr_session, evdev_pointer_button};
+
+/// One in-flight command from the public API to the owner thread.
+enum Cmd {
+ Press {
+ cursor_id: String,
+ window_id: u64,
+ x: i32,
+ y: i32,
+ button: u8,
+ reply: Sender<anyhow::Result<()>>,
+ },
+ MoveTo {
+ cursor_id: String,
+ x: i32,
+ y: i32,
+ reply: Sender<anyhow::Result<()>>,
+ },
+ Release {
+ cursor_id: String,
+ button: u8,
+ reply: Sender<anyhow::Result<()>>,
+ },
+ /// Drop the entry for a cursor_id without sending wire events — used to
+ /// recover when the compositor disconnected mid-life.
+ Forget {
+ cursor_id: String,
+ reply: Sender<anyhow::Result<()>>,
+ },
+}
+
+/// State held inside the owner thread for one cursor_id.
+struct ActivePointer {
+ vptr: ZwlrVirtualPointerV1,
+ /// evdev codes of buttons currently held down. When this set becomes
+ /// empty the vptr is destroyed and the entry dropped from the map.
+ held: HashSet<u32>,
+ /// Output extent at session open time — needed for motion_absolute.
+ out_w: u32,
+ out_h: u32,
+}
+
+/// Process-global command channel into the owner thread. Lazily started on
+/// first use.
+static TX: OnceLock<Sender<Cmd>> = OnceLock::new();
+
+fn tx() -> &'static Sender<Cmd> {
+ TX.get_or_init(|| {
+ let (tx, rx) = bounded::<Cmd>(32);
+ thread::Builder::new()
+ .name("cua-persistent-vptr".into())
+ .spawn(move || owner_thread(rx))
+ .expect("spawn cua-persistent-vptr thread");
+ tx
+ })
+}
+
+fn owner_thread(rx: Receiver<Cmd>) {
+ let mut active: HashMap<String, ActivePointer> = HashMap::new();
+ while let Ok(cmd) = rx.recv() {
+ match cmd {
+ Cmd::Press { cursor_id, window_id, x, y, button, reply } => {
+ let r = handle_press(&mut active, &cursor_id, window_id, x, y, button);
+ let _ = reply.send(r);
+ }
+ Cmd::MoveTo { cursor_id, x, y, reply } => {
+ let r = handle_move(&mut active, &cursor_id, x, y);
+ let _ = reply.send(r);
+ }
+ Cmd::Release { cursor_id, button, reply } => {
+ let r = handle_release(&mut active, &cursor_id, button);
+ let _ = reply.send(r);
+ }
+ Cmd::Forget { cursor_id, reply } => {
+ if let Some(p) = active.remove(&cursor_id) {
+ p.vptr.destroy();
+ }
+ let _ = reply.send(Ok(()));
+ }
+ }
+ }
+}
+
+fn handle_press(
+ active: &mut HashMap<String, ActivePointer>,
+ cursor_id: &str,
+ window_id: u64,
+ x: i32,
+ y: i32,
+ button: u8,
+) -> anyhow::Result<()> {
+ // Open a fresh session for this press — this binds the seat, the foreign-
+ // toplevel manager, activates the target window, and creates a new vptr.
+ // Keep the (out_w, out_h) but drop the queue + state at end of scope; the
+ // vptr itself remains alive (Wayland objects survive their original queue
+ // as long as the Connection is alive).
+ let mut sess = open_vptr_session(Some(window_id as u32))?;
+ let (w, h) = (sess.output_w, sess.output_h);
+ let px = x.clamp(0, w as i32 - 1) as u32;
+ let py = y.clamp(0, h as i32 - 1) as u32;
+ let btn = evdev_pointer_button(button);
+
+ sess.vptr.motion_absolute(0, px, py, w, h);
+ sess.vptr.frame();
+ sess.vptr.button(0, btn, ButtonState::Pressed);
+ sess.vptr.frame();
+ sess.queue.roundtrip(&mut sess.state)?;
+
+ // Take ownership of the vptr handle by extracting it from the session.
+ // ZwlrVirtualPointerV1 is a Wayland proxy — cloning it gives another
+ // handle to the same wire object; destroying it sends the destructor.
+ let vptr = sess.vptr.clone();
+ // Persist the live connection so the proxy stays valid after this fn returns
+ // (the session goes out of scope; we need the conn alive).
+ // We do this by leaking the connection into a process-static slot keyed by
+ // cursor_id. Subsequent commands on the same cursor reuse this conn.
+ persist_conn(cursor_id, sess.conn);
+
+ let mut held = HashSet::new();
+ held.insert(btn);
+ active.insert(cursor_id.to_string(), ActivePointer { vptr, held, out_w: w, out_h: h });
+ Ok(())
+}
+
+fn handle_move(
+ active: &mut HashMap<String, ActivePointer>,
+ cursor_id: &str,
+ x: i32,
+ y: i32,
+) -> anyhow::Result<()> {
+ let entry = active.get_mut(cursor_id).ok_or_else(|| {
+ anyhow::anyhow!("no held mouse button for cursor '{cursor_id}'; call mouse_button_down first")
+ })?;
+ let px = x.clamp(0, entry.out_w as i32 - 1) as u32;
+ let py = y.clamp(0, entry.out_h as i32 - 1) as u32;
+ entry.vptr.motion_absolute(0, px, py, entry.out_w, entry.out_h);
+ entry.vptr.frame();
+ roundtrip_on_persistent(cursor_id)?;
+ Ok(())
+}
+
+fn handle_release(
+ active: &mut HashMap<String, ActivePointer>,
+ cursor_id: &str,
+ button: u8,
+) -> anyhow::Result<()> {
+ let btn = evdev_pointer_button(button);
+ let drop_entry = {
+ let entry = active.get_mut(cursor_id).ok_or_else(|| {
+ anyhow::anyhow!("no held mouse button for cursor '{cursor_id}'")
+ })?;
+ entry.vptr.button(0, btn, ButtonState::Released);
+ entry.vptr.frame();
+ roundtrip_on_persistent(cursor_id)?;
+ entry.held.remove(&btn);
+ entry.held.is_empty()
+ };
+ if drop_entry {
+ if let Some(p) = active.remove(cursor_id) {
+ p.vptr.destroy();
+ roundtrip_on_persistent(cursor_id).ok();
+ }
+ forget_conn(cursor_id);
+ }
+ Ok(())
+}
+
+// Process-static slots for Connection + EventQueue keyed by cursor_id. The
+// EventQueue is !Send but we only touch these on the owner thread, so wrap
+// in a thread-local-by-construction pattern: store inside the same map so
+// the owner thread is the sole accessor.
+//
+// We use a per-thread static rather than a Mutex<HashMap> because the owner
+// thread is the only accessor (no contention possible).
+thread_local! {
+ static CONNS: std::cell::RefCell<HashMap<String, (Connection, wayland_client::EventQueue<super::State>)>>
+ = std::cell::RefCell::new(HashMap::new());
+}
+
+fn persist_conn(cursor_id: &str, conn: Connection) {
+ let queue = conn.new_event_queue::<super::State>();
+ CONNS.with(|c| { c.borrow_mut().insert(cursor_id.to_string(), (conn, queue)); });
+}
+
+fn forget_conn(cursor_id: &str) {
+ CONNS.with(|c| { c.borrow_mut().remove(cursor_id); });
+}
+
+fn roundtrip_on_persistent(cursor_id: &str) -> anyhow::Result<()> {
+ CONNS.with(|c| {
+ let mut b = c.borrow_mut();
+ let (_conn, queue) = b.get_mut(cursor_id)
+ .ok_or_else(|| anyhow::anyhow!("no persistent connection for cursor '{cursor_id}'"))?;
+ let mut tmp = super::State::default();
+ queue.roundtrip(&mut tmp).map_err(|e| anyhow::anyhow!("compositor roundtrip failed: {e}"))?;
+ Ok(())
+ })
+}
+
+// ── public API ────────────────────────────────────────────────────────────
+
+/// Press and HOLD `button` (evdev code) at output coordinates `(x, y)` on the
+/// toplevel identified by `window_id`. Subsequent `move_to` / `release` calls
+/// targeting the same `cursor_id` reuse the same virtual-pointer device, so
+/// the compositor treats the sequence as one logical drag rather than as
+/// independent clicks. Errors if `cursor_id` already has a held button.
+pub fn press(cursor_id: &str, window_id: u64, x: i32, y: i32, button: u8) -> anyhow::Result<()> {
+ let (tx_r, rx_r) = bounded(1);
+ tx().send(Cmd::Press {
+ cursor_id: cursor_id.to_string(),
+ window_id, x, y, button,
+ reply: tx_r,
+ }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
+ rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
+}
+
+/// Emit motion_absolute on the held cursor's virtual-pointer. Errors if there
+/// is no held button for `cursor_id`.
+pub fn move_to(cursor_id: &str, x: i32, y: i32) -> anyhow::Result<()> {
+ let (tx_r, rx_r) = bounded(1);
+ tx().send(Cmd::MoveTo {
+ cursor_id: cursor_id.to_string(), x, y, reply: tx_r,
+ }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
+ rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
+}
+
+/// Release `button` on the held cursor. If no other buttons remain held the
+/// virtual-pointer is destroyed and its Wayland connection torn down.
+pub fn release(cursor_id: &str, button: u8) -> anyhow::Result<()> {
+ let (tx_r, rx_r) = bounded(1);
+ tx().send(Cmd::Release {
+ cursor_id: cursor_id.to_string(), button, reply: tx_r,
+ }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
+ rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
+}
+
+/// Drop the entry for `cursor_id` without emitting any Wayland events.
+/// Useful for recovery — if the agent thinks a button is held but the
+/// compositor disagrees, this clears the local state without trying to
+/// send a release that would error.
+pub fn forget(cursor_id: &str) -> anyhow::Result<()> {
+ let (tx_r, rx_r) = bounded(1);
+ tx().send(Cmd::Forget {
+ cursor_id: cursor_id.to_string(), reply: tx_r,
+ }).map_err(|e| anyhow::anyhow!("cua-persistent-vptr thread is dead: {e}"))?;
+ rx_r.recv().map_err(|e| anyhow::anyhow!("reply channel closed: {e}"))?
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rs
new file mode 100644
index 0000000000..cba0dd6c00
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screencast.rs
@@ -0,0 +1,444 @@
+//! Per-window capture via `xdg-desktop-portal` ScreenCast + PipeWire.
+//!
+//! Reaches GNOME/Mutter and KDE/KWin per-window selection — the
+//! cross-DE complement to wlroots `ext-image-copy-capture-v1`. Sequence:
+//!
+//! 1. ashpd ScreenCast::create_session.
+//! 2. select_sources with types=Window|Monitor, cursor_mode=Embedded,
+//! persist_mode=Application (consent dialog fires ONCE per process,
+//! cached for the lifetime of the requesting binary).
+//! 3. start(session, parent_window) — user picks the source window in
+//! the portal's system dialog the first time; subsequent calls within
+//! the same process reuse the cached choice.
+//! 4. open_pipe_wire_remote returns a Unix file descriptor; Streams
+//! carries the published node ids.
+//! 5. Connect a pipewire client to the fd, open a Stream targeting the
+//! advertised node id, negotiate Video/Raw format, dequeue one frame
+//! with `STREAM_FLAG_MAP_BUFFERS`, copy pixels, PNG-encode, stop.
+//!
+//! The session + restore_token are persisted in a process-local
+//! `OnceLock<PortalScreencastSession>` so only the first call across the
+//! process lifetime triggers the consent dialog; later calls reuse the
+//! same PipeWire node id.
+
+use std::os::fd::AsRawFd;
+use std::sync::OnceLock;
+
+use ashpd::desktop::screencast::{
+ CursorMode, OpenPipeWireRemoteOptions, Screencast,
+ SelectSourcesOptions, SourceType, StartCastOptions, Streams,
+};
+use ashpd::desktop::{CreateSessionOptions, PersistMode, Session};
+use ashpd::enumflags2::BitFlags;
+
+/// A live portal ScreenCast session held across calls. Expensive to
+/// create (system consent dialog) and cheap to reuse — the OnceLock
+/// keeps it alive until the requesting process exits.
+struct PortalScreencastSession {
+ _session: Session<Screencast>,
+ streams: Streams,
+ fd: std::os::fd::OwnedFd,
+}
+
+unsafe impl Send for PortalScreencastSession {}
+unsafe impl Sync for PortalScreencastSession {}
+
+static SESSION: OnceLock<PortalScreencastSession> = OnceLock::new();
+
+/// Capture the user-selected window via `xdg-desktop-portal` ScreenCast.
+/// On first invocation per process the portal dialog asks the user to
+/// pick a window or monitor; subsequent calls reuse the same node
+/// transparently. Returns PNG bytes of the latest frame on the stream.
+pub fn screenshot_window_via_portal() -> anyhow::Result<Vec<u8>> {
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?;
+
+ if SESSION.get().is_none() {
+ let sess = rt.block_on(create_portal_session())?;
+ let _ = SESSION.set(sess);
+ }
+ let session = SESSION
+ .get()
+ .ok_or_else(|| anyhow::anyhow!("portal ScreenCast session unavailable"))?;
+
+ let node_id = session
+ .streams
+ .streams()
+ .iter()
+ .next()
+ .map(|s| s.pipe_wire_node_id())
+ .ok_or_else(|| anyhow::anyhow!("portal advertised no streams"))?;
+
+ capture_one_frame(session.fd.as_raw_fd(), node_id)
+}
+
+async fn create_portal_session() -> anyhow::Result<PortalScreencastSession> {
+ let proxy = Screencast::new()
+ .await
+ .map_err(|e| anyhow::anyhow!("portal ScreenCast proxy unreachable: {e}. Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde."))?;
+
+ let session = proxy
+ .create_session(CreateSessionOptions::default())
+ .await
+ .map_err(|e| anyhow::anyhow!("portal create_session failed: {e}"))?;
+
+ let select_opts = SelectSourcesOptions::default()
+ .set_sources(BitFlags::<SourceType>::from(SourceType::Window) | SourceType::Monitor)
+ .set_cursor_mode(CursorMode::Embedded)
+ .set_persist_mode(PersistMode::Application);
+
+ proxy
+ .select_sources(&session, select_opts)
+ .await
+ .map_err(|e| anyhow::anyhow!("portal select_sources failed: {e}"))?
+ .response()
+ .map_err(|e| anyhow::anyhow!("portal select_sources response error: {e}"))?;
+
+ let streams = proxy
+ .start(&session, None, StartCastOptions::default())
+ .await
+ .map_err(|e| anyhow::anyhow!("portal start failed: {e}"))?
+ .response()
+ .map_err(|e| anyhow::anyhow!("portal start response error (user denied?): {e}"))?;
+
+ let fd = proxy
+ .open_pipe_wire_remote(&session, OpenPipeWireRemoteOptions::default())
+ .await
+ .map_err(|e| anyhow::anyhow!("portal open_pipe_wire_remote failed: {e}"))?;
+
+ Ok(PortalScreencastSession {
+ _session: session,
+ streams,
+ fd,
+ })
+}
+
+/// Pull one frame off the PipeWire stream advertised by the portal,
+/// encode it as PNG, and return the bytes.
+///
+/// Pipeline:
+/// 1. `pipewire::MainLoop` + `Context` + `Core::connect_fd(portal_fd)` —
+/// join the portal-published PipeWire remote on the same fd ashpd
+/// returned from `open_pipe_wire_remote`.
+/// 2. `Stream::new` with `media.type=Video, media.category=Capture,
+/// media.role=Screen` so PipeWire knows to route the cast properly.
+/// 3. Build an `EnumFormat` SPA pod listing the channel-orders we accept
+/// (BGRx, BGRA, RGBx, RGBA) at any size up to 8K — the compositor
+/// picks one and announces it via `param_changed`. We track w/h/fmt
+/// from that callback.
+/// 4. The `process` callback dequeues exactly one buffer, copies the
+/// pixels out into a shared `Arc<Mutex<...>>`, and quits the main
+/// loop. Subsequent buffers are dropped (queued back via Buffer::Drop)
+/// once `frame.is_some()`.
+/// 5. After `mainloop.run()` returns, channel-swap the captured bytes
+/// into RGBA and PNG-encode them via the `image` crate.
+fn capture_one_frame(pipewire_fd: i32, node_id: u32) -> anyhow::Result<Vec<u8>> {
+ use std::os::fd::FromRawFd;
+ use std::sync::{Arc, Mutex};
+
+ use pipewire as pw;
+ use pw::{properties::properties, spa};
+ use spa::pod::Pod;
+
+ pw::init();
+
+ let mainloop = pw::main_loop::MainLoop::new(None)
+ .map_err(|e| anyhow::anyhow!("pipewire MainLoop::new failed: {e}"))?;
+ let context = pw::context::Context::new(&mainloop)
+ .map_err(|e| anyhow::anyhow!("pipewire Context::new failed: {e}"))?;
+
+ // dup the portal fd so the OwnedFd we hand to connect_fd has an
+ // independent lifecycle from the PortalScreencastSession's master copy.
+ let dup_fd = unsafe { libc::dup(pipewire_fd) };
+ if dup_fd < 0 {
+ anyhow::bail!(
+ "dup of portal PipeWire fd failed: {}",
+ std::io::Error::last_os_error()
+ );
+ }
+ let owned_fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(dup_fd) };
+ let core = context
+ .connect_fd(owned_fd, None)
+ .map_err(|e| anyhow::anyhow!("pipewire connect_fd failed: {e}"))?;
+
+ // Shared state populated by the param_changed + process callbacks.
+ // FrameBuf carries the negotiated geometry alongside the pixel data so
+ // the PNG encoder picks the right stride / channel order on exit.
+ struct FrameBuf {
+ bytes: Vec<u8>,
+ width: u32,
+ height: u32,
+ stride: u32,
+ format: spa::param::video::VideoFormat,
+ }
+ let frame: Arc<Mutex<Option<FrameBuf>>> = Arc::new(Mutex::new(None));
+ let neg_format: Arc<Mutex<spa::param::video::VideoInfoRaw>> =
+ Arc::new(Mutex::new(Default::default()));
+
+ let stream = pw::stream::Stream::new(
+ &core,
+ "cua-driver-capture",
+ properties! {
+ *pw::keys::MEDIA_TYPE => "Video",
+ *pw::keys::MEDIA_CATEGORY => "Capture",
+ *pw::keys::MEDIA_ROLE => "Screen",
+ },
+ )
+ .map_err(|e| anyhow::anyhow!("pipewire Stream::new failed: {e}"))?;
+
+ let mainloop_for_process = mainloop.clone();
+ let neg_format_for_param = neg_format.clone();
+ let frame_for_process = frame.clone();
+ let neg_format_for_process = neg_format.clone();
+
+ let _listener = stream
+ .add_local_listener::<()>()
+ .param_changed(move |_stream, _user, id, param| {
+ // The compositor confirms the negotiated VideoInfoRaw via the
+ // Format param event right after stream.connect — capture it
+ // so the process callback knows the width/height/format.
+ let Some(param) = param else { return };
+ if id != spa::param::ParamType::Format.as_raw() {
+ return;
+ }
+ let (media_type, media_subtype) = match spa::param::format_utils::parse_format(param) {
+ Ok(v) => v,
+ Err(_) => return,
+ };
+ if media_type != spa::param::format::MediaType::Video
+ || media_subtype != spa::param::format::MediaSubtype::Raw
+ {
+ return;
+ }
+ let mut info = spa::param::video::VideoInfoRaw::default();
+ if info.parse(param).is_ok() {
+ if let Ok(mut g) = neg_format_for_param.lock() {
+ *g = info;
+ }
+ }
+ })
+ .process(move |stream, _user| {
+ // Already captured? Drop further buffers (they'll be queued
+ // back via Buffer::Drop) so the run loop can finish.
+ if frame_for_process.lock().ok().map(|g| g.is_some()).unwrap_or(false) {
+ let _ = stream.dequeue_buffer();
+ return;
+ }
+ let mut buffer = match stream.dequeue_buffer() {
+ Some(b) => b,
+ None => return,
+ };
+ let datas = buffer.datas_mut();
+ if datas.is_empty() {
+ return;
+ }
+ let info = match neg_format_for_process.lock() {
+ Ok(g) => *g,
+ Err(_) => return,
+ };
+ let size_struct = info.size();
+ let width = size_struct.width;
+ let height = size_struct.height;
+ if width == 0 || height == 0 {
+ return;
+ }
+ let chunk_stride = datas[0].chunk().stride();
+ let chunk_size = datas[0].chunk().size();
+ // PipeWire sometimes leaves stride at 0 (when the producer
+ // doesn't fill it in); fall back to width*4 for the packed
+ // BGRx/BGRA/RGBx/RGBA formats we negotiate.
+ let stride = if chunk_stride > 0 {
+ chunk_stride as u32
+ } else if chunk_size > 0 && height > 0 {
+ chunk_size / height
+ } else {
+ width * 4
+ };
+ let payload = match datas[0].data() {
+ Some(p) => p,
+ None => return,
+ };
+ let payload_len = (stride as usize) * (height as usize);
+ if payload.len() < payload_len {
+ return;
+ }
+ let bytes = payload[..payload_len].to_vec();
+ if let Ok(mut g) = frame_for_process.lock() {
+ *g = Some(FrameBuf {
+ bytes,
+ width,
+ height,
+ stride,
+ format: info.format(),
+ });
+ }
+ // Quit the main loop so the caller can encode + return.
+ mainloop_for_process.quit();
+ })
+ .register()
+ .map_err(|e| anyhow::anyhow!("pipewire stream listener register failed: {e}"))?;
+
+ // Build an EnumFormat SPA pod listing the formats + sizes we accept.
+ // Compositors typically pick BGRx/BGRA on Linux desktops; we list a
+ // few alternates so negotiation succeeds on more sources.
+ let obj = pw::spa::pod::object!(
+ pw::spa::utils::SpaTypes::ObjectParamFormat,
+ pw::spa::param::ParamType::EnumFormat,
+ pw::spa::pod::property!(
+ pw::spa::param::format::FormatProperties::MediaType,
+ Id,
+ pw::spa::param::format::MediaType::Video
+ ),
+ pw::spa::pod::property!(
+ pw::spa::param::format::FormatProperties::MediaSubtype,
+ Id,
+ pw::spa::param::format::MediaSubtype::Raw
+ ),
+ pw::spa::pod::property!(
+ pw::spa::param::format::FormatProperties::VideoFormat,
+ Choice,
+ Enum,
+ Id,
+ pw::spa::param::video::VideoFormat::BGRx,
+ pw::spa::param::video::VideoFormat::BGRx,
+ pw::spa::param::video::VideoFormat::BGRA,
+ pw::spa::param::video::VideoFormat::RGBx,
+ pw::spa::param::video::VideoFormat::RGBA,
+ ),
+ pw::spa::pod::property!(
+ pw::spa::param::format::FormatProperties::VideoSize,
+ Choice,
+ Range,
+ Rectangle,
+ pw::spa::utils::Rectangle { width: 1920, height: 1080 },
+ pw::spa::utils::Rectangle { width: 1, height: 1 },
+ pw::spa::utils::Rectangle { width: 7680, height: 4320 }
+ ),
+ pw::spa::pod::property!(
+ pw::spa::param::format::FormatProperties::VideoFramerate,
+ Choice,
+ Range,
+ Fraction,
+ pw::spa::utils::Fraction { num: 30, denom: 1 },
+ pw::spa::utils::Fraction { num: 0, denom: 1 },
+ pw::spa::utils::Fraction { num: 1000, denom: 1 }
+ ),
+ );
+ let values: Vec<u8> = pw::spa::pod::serialize::PodSerializer::serialize(
+ std::io::Cursor::new(Vec::new()),
+ &pw::spa::pod::Value::Object(obj),
+ )
+ .map_err(|e| anyhow::anyhow!("EnumFormat pod serialization failed: {e}"))?
+ .0
+ .into_inner();
+ let mut params = [Pod::from_bytes(&values)
+ .ok_or_else(|| anyhow::anyhow!("EnumFormat pod parse-back failed"))?];
+
+ stream
+ .connect(
+ spa::utils::Direction::Input,
+ Some(node_id),
+ pw::stream::StreamFlags::AUTOCONNECT | pw::stream::StreamFlags::MAP_BUFFERS,
+ &mut params,
+ )
+ .map_err(|e| anyhow::anyhow!("pipewire stream.connect failed: {e}"))?;
+
+ // Install a 10-second timeout source so a producer that never delivers
+ // a frame (eg the user closed the consent dialog mid-cast) doesn't
+ // hang the main loop forever.
+ let timeout_loop = mainloop.clone();
+ let _timeout_source = mainloop
+ .loop_()
+ .add_timer(move |_expirations: u64| timeout_loop.quit());
+ _timeout_source
+ .update_timer(Some(std::time::Duration::from_secs(10)), None)
+ .into_result()
+ .map_err(|e| anyhow::anyhow!("pipewire timeout source failed: {e:?}"))?;
+
+ mainloop.run();
+
+ // Tear down the stream before we read the frame, so the producer
+ // can't keep mutating the buffer underneath us.
+ drop(_listener);
+ drop(stream);
+
+ let frame = frame
+ .lock()
+ .map_err(|_| anyhow::anyhow!("frame mutex poisoned"))?
+ .take()
+ .ok_or_else(|| anyhow::anyhow!("portal ScreenCast: no frame arrived within timeout"))?;
+
+ encode_frame_to_png(&frame.bytes, frame.width, frame.height, frame.stride, frame.format)
+}
+
+/// Channel-swap a PipeWire video frame into RGBA8888 and PNG-encode it.
+///
+/// Mirrors `ext_screencopy::encode_buffer_to_png` — the input layout
+/// depends on the negotiated VideoFormat: BGRx/BGRA are little-endian
+/// BGRA (so swap R↔B), RGBx/RGBA are already RGBA in memory.
+fn encode_frame_to_png(
+ pixels: &[u8],
+ width: u32,
+ height: u32,
+ stride: u32,
+ fmt: libspa::param::video::VideoFormat,
+) -> anyhow::Result<Vec<u8>> {
+ use libspa::param::video::VideoFormat;
+ let mut rgba: Vec<u8> = Vec::with_capacity((width as usize) * (height as usize) * 4);
+ for y in 0..(height as usize) {
+ let row_start = y * (stride as usize);
+ for x in 0..(width as usize) {
+ let i = row_start + x * 4;
+ let (r, g, b, a) = match fmt {
+ VideoFormat::BGRx => (pixels[i + 2], pixels[i + 1], pixels[i], 0xFF),
+ VideoFormat::BGRA => (pixels[i + 2], pixels[i + 1], pixels[i], pixels[i + 3]),
+ VideoFormat::RGBx => (pixels[i], pixels[i + 1], pixels[i + 2], 0xFF),
+ VideoFormat::RGBA => (pixels[i], pixels[i + 1], pixels[i + 2], pixels[i + 3]),
+ // Fall through: treat unknown formats as BGRA (the most
+ // common compositor output on Linux).
+ _ => (pixels[i + 2], pixels[i + 1], pixels[i], pixels[i + 3]),
+ };
+ rgba.extend_from_slice(&[r, g, b, a]);
+ }
+ }
+ use image::{codecs::png::PngEncoder, ImageBuffer, ImageEncoder, Rgba};
+ let img: ImageBuffer<Rgba<u8>, _> =
+ ImageBuffer::from_raw(width, height, rgba).ok_or_else(|| {
+ anyhow::anyhow!("internal: buffer dims mismatch ({}x{})", width, height)
+ })?;
+ let mut out: Vec<u8> = Vec::new();
+ PngEncoder::new(&mut out).write_image(
+ img.as_raw(),
+ width,
+ height,
+ image::ExtendedColorType::Rgba8,
+ )?;
+ Ok(out)
+}
+
+/// Probe whether the portal ScreenCast interface is reachable on the
+/// session bus without taking an actual capture (avoids cached consent).
+pub fn probe_screencast_portal() -> anyhow::Result<bool> {
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime: {e}"))?;
+ rt.block_on(async {
+ let conn = zbus::Connection::session()
+ .await
+ .map_err(|e| anyhow::anyhow!("session bus unreachable: {e}"))?;
+ let proxy = zbus::fdo::DBusProxy::new(&conn)
+ .await
+ .map_err(|e| anyhow::anyhow!("dbus proxy creation failed: {e}"))?;
+ let bus_name: zbus::names::BusName = "org.freedesktop.portal.Desktop"
+ .try_into()
+ .map_err(|e| anyhow::anyhow!("bus name parse failed: {e}"))?;
+ let has_owner = proxy
+ .name_has_owner(bus_name)
+ .await
+ .map_err(|e| anyhow::anyhow!("name_has_owner failed: {e}"))?;
+ Ok(has_owner)
+ })
+}
diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs
new file mode 100644
index 0000000000..c1f1afbc0a
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs
@@ -0,0 +1,124 @@
+//! `xdg-desktop-portal` Screenshot capture path.
+//!
+//! Reaches GNOME/Mutter, KDE/KWin, and any compositor that ships an
+//! xdg-desktop-portal backend — superset of the wlroots-only
+//! `zwlr_screencopy_manager_v1` path. The portal asks the user for consent
+//! on first invocation per session (GNOME caches via `Settings → Privacy →
+//! Screenshot`; KDE Plasma 6 caches per requesting binary); subsequent
+//! calls in the same session are silent.
+//!
+//! ashpd is async (zbus under the hood); the rest of `wayland::screenshot_*`
+//! is sync because it's called from `tokio::task::spawn_blocking`. We build
+//! a dedicated single-threaded tokio runtime per call so the async ashpd
+//! work has somewhere to live without colliding with the caller's runtime.
+
+use std::path::PathBuf;
+
+use ashpd::desktop::screenshot::Screenshot;
+
+/// Capture the current display via `org.freedesktop.portal.Screenshot.Screenshot`.
+/// Returns the PNG/JPEG bytes the portal wrote to disk, or a typed error
+/// when the portal isn't reachable (e.g. wlroots compositor without
+/// `xdg-desktop-portal-wlr`, KDE without `xdg-desktop-portal-kde` running,
+/// or no D-Bus session bus).
+///
+/// `interactive=false` skips the customization dialog (region select,
+/// post-shot editor) — the user still consents to the screenshot itself
+/// on the first call per session unless they've pre-approved the
+/// requesting binary in their DE's privacy settings.
+pub fn screenshot_via_portal() -> anyhow::Result<Vec<u8>> {
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?;
+ let uri = rt.block_on(async {
+ Screenshot::request()
+ .interactive(false)
+ .modal(false)
+ .send()
+ .await
+ .map_err(map_ashpd_err)?
+ .response()
+ .map_err(map_ashpd_err)
+ .map(|resp| resp.uri().clone())
+ })?;
+
+ // The portal returns a file:// URI pointing at a PNG (typically in
+ // /tmp or $XDG_RUNTIME_DIR/doc/...). Decode it via the `url` crate.
+ let path = uri_to_path(&uri.to_string())
+ .ok_or_else(|| anyhow::anyhow!("portal returned a non-file URI: {uri}"))?;
+ let bytes = std::fs::read(&path)
+ .map_err(|e| anyhow::anyhow!("portal screenshot at {} unreadable: {e}", path.display()))?;
+ // Best-effort cleanup: the portal places the file in a process-readable
+ // location that can outlive our process. Removing avoids the disk
+ // accumulating one PNG per call. If removal fails (permissions, file
+ // already gone) we silently drop the error — the bytes are already
+ // captured.
+ let _ = std::fs::remove_file(&path);
+ Ok(bytes)
+}
+
+/// Convert a `file://...` URI string to a local PathBuf. Returns None for
+/// non-file schemes (e.g. portal `doc://...` URIs in some sandboxed
+/// configurations, which would need a second resolution step through the
+/// document portal).
+fn uri_to_path(uri_str: &str) -> Option<PathBuf> {
+ let url = url::Url::parse(uri_str).ok()?;
+ if url.scheme() != "file" {
+ return None;
+ }
+ url.to_file_path().ok()
+}
+
+fn map_ashpd_err<E: std::fmt::Display>(e: E) -> anyhow::Error {
+ let msg = format!("xdg-desktop-portal Screenshot failed: {e}");
+ // Pattern-match common failure modes to produce a typed-ish error
+ // the caller can pivot on.
+ if msg.contains("ServiceUnknown") || msg.contains("NotFound") {
+ anyhow::anyhow!("xdg-desktop-portal is not running on this session ({msg}). Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde / xdg-desktop-portal-wlr for your compositor.")
+ } else if msg.contains("Cancelled") || msg.contains("denied") {
+ anyhow::anyhow!("xdg-desktop-portal Screenshot consent dialog was cancelled or denied: {msg}")
+ } else {
+ anyhow::anyhow!(msg)
+ }
+}
+
+/// Probe whether the portal *service* is reachable on the session bus.
+/// Returns Ok(true) when the `org.freedesktop.portal.Desktop` name has an
+/// owner, Ok(false) otherwise, and Err(...) for transport failures
+/// (no session bus, etc.).
+///
+/// Read-only and side-effect free — does NOT actually invoke the
+/// Screenshot interface, so it doesn't cache a consent grant on GNOME /
+/// KDE backends that gate Screenshot on user prompts. The doctor uses
+/// this to surface portal availability without burning an interactive
+/// dialog every time `hermes computer-use doctor` runs.
+///
+/// Note: a true return here means the portal *service* is present. The
+/// Screenshot *interface* on that service may still be missing or
+/// disabled — for that, an actual `Screenshot::request().send().await`
+/// is needed. We accept the false-positive risk in exchange for the
+/// no-consent, no-burn-dialog property.
+pub fn probe_portal() -> anyhow::Result<bool> {
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd probe: {e}"))?;
+ rt.block_on(async {
+ // Cheapest possible probe: open a session bus and check that the
+ // portal name has an owner. We avoid Screenshot::request() because
+ // that would cache a (potentially undesired) consent grant.
+ let conn = zbus::Connection::session()
+ .await
+ .map_err(|e| anyhow::anyhow!("session bus unreachable: {e}"))?;
+ let proxy = zbus::fdo::DBusProxy::new(&conn)
+ .await
+ .map_err(|e| anyhow::anyhow!("dbus proxy creation failed: {e}"))?;
+ let has_owner = proxy
+ .name_has_owner("org.freedesktop.portal.Desktop".try_into()
+ .map_err(|e| anyhow::anyhow!("bus name parse failed: {e}"))?)
+ .await
+ .map_err(|e| anyhow::anyhow!("name_has_owner failed: {e}"))?;
+ Ok(has_owner)
+ })
+}
diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nix
index 30cb7e39f9..409f163c86 100644
--- a/nix/cua-driver/package.nix
+++ b/nix/cua-driver/package.nix
@@ -46,14 +46,31 @@ pkgs.rustPlatform.buildRustPackage {
# ureq -> rustls (no openssl)
# tiny-skia -> pure Rust 2D graphics
# ring -> compiles own C/asm via stdenv's cc
- # Except the `x11` crate (raw Xlib FFI for MPX multi-cursor drags), whose
- # build.rs locates libX11/libXi/libXtst via pkg-config.
- nativeBuildInputs = [ pkgs.pkg-config ];
+ # The `x11` crate (raw Xlib FFI for MPX multi-cursor drags) needs
+ # libX11/libXi/libXtst via pkg-config. The Wayland-parity work
+ # (round 2/3 — wayland::portal_screencast / wayland::libei) adds:
+ # pipewire 0.8 -> needs libpipewire-0.3 + libspa headers via pkg-config
+ # + bindgen (clang) for SPA pod generation
+ # reis 0.7 -> pure Rust libei binding, but the workspace pulls
+ # libei-dev transitively through ashpd's tokio feature
+ nativeBuildInputs = with pkgs; [
+ pkg-config
+ # bindgen needs a libclang at build time for the libpipewire / libspa
+ # SPA pod codegen. rust-bindgen also picks up `stdbool.h` etc. from
+ # clang's resource dir, not glibc.
+ rustPlatform.bindgenHook
+ ];
buildInputs = with pkgs; [
libx11
libxi
libxtst
libxext
+ # Wayland-parity additions: PipeWire is needed by the portal
+ # ScreenCast capture path (wayland::portal_screencast). pipewire
+ # already pulls libspa transitively in nixpkgs.
+ pipewire
+ # libei via reis — same as the ashpd RemoteDesktop+EIS flow.
+ libei
];
# Skip tests that require a running X11 display or AT-SPI bus