Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(windows): background input without z-raise + multi-cursor demos + overlay improvements (#1809)

ddupont53bb84cc71cb
diff --git a/.gitignore b/.gitignore
index 32a250ed36..a91bb24b5e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -217,4 +217,6 @@ post-provision
scripts/check-repo-md-links.py
docs/scripts/check-links.py
docs/scripts/check-all-links.py
-docs/scripts/check-mdx-links.py
\ No newline at end of file
+docs/scripts/check-mdx-links.py
+# Local Windows RE scratch (downloaded PDBs, disassembly scripts)
+.re-windows/
diff --git a/.vscode/settings.json b/.vscode/settings.json
index c06f93aadf..c2db3d0c77 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -21,5 +21,18 @@
],
"mypy-type-checker.path": [
"${workspaceFolder}"
- ]
+ ],
+ "workbench.colorCustomizations": {
+ "activityBar.background": "#410A56",
+ "titleBar.activeBackground": "#5A0E78",
+ "titleBar.activeForeground": "#FEFBFF",
+ "titleBar.inactiveBackground": "#410A56",
+ "titleBar.inactiveForeground": "#FEFBFF",
+ "statusBar.background": "#410A56",
+ "statusBar.foreground": "#FEFBFF",
+ "statusBar.debuggingBackground": "#410A56",
+ "statusBar.debuggingForeground": "#FEFBFF",
+ "statusBar.noFolderBackground": "#410A56",
+ "statusBar.noFolderForeground": "#FEFBFF"
+ }
}
\ No newline at end of file
diff --git a/demo/jukebox/Cargo.lock b/demo/jukebox/Cargo.lock
new file mode 100644
index 0000000000..10a42677ce
--- /dev/null
+++ b/demo/jukebox/Cargo.lock
@@ -0,0 +1,1072 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "aho-corasick"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "alsa"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43"
+dependencies = [
+ "alsa-sys",
+ "bitflags 2.11.1",
+ "cfg-if",
+ "libc",
+]
+
+[[package]]
+name = "alsa-sys"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527"
+dependencies = [
+ "libc",
+ "pkg-config",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "bindgen"
+version = "0.72.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"
+dependencies = [
+ "bitflags 2.11.1",
+ "cexpr",
+ "clang-sys",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex 1.3.0",
+ "syn",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bitflags"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
+
+[[package]]
+name = "bumpalo"
+version = "3.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
+
+[[package]]
+name = "bytes"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
+
+[[package]]
+name = "cc"
+version = "1.2.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
+dependencies = [
+ "find-msvc-tools",
+ "jobserver",
+ "libc",
+ "shlex 2.0.1",
+]
+
+[[package]]
+name = "cesu8"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
+
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
+[[package]]
+name = "combine"
+version = "4.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
+dependencies = [
+ "bytes",
+ "memchr",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
+
+[[package]]
+name = "coreaudio-rs"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation-sys",
+ "coreaudio-sys",
+]
+
+[[package]]
+name = "coreaudio-sys"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ceec7a6067e62d6f931a2baf6f3a751f4a892595bcec1461a3c94ef9949864b6"
+dependencies = [
+ "bindgen",
+]
+
+[[package]]
+name = "cpal"
+version = "0.15.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779"
+dependencies = [
+ "alsa",
+ "core-foundation-sys",
+ "coreaudio-rs",
+ "dasp_sample",
+ "jni",
+ "js-sys",
+ "libc",
+ "mach2",
+ "ndk",
+ "ndk-context",
+ "oboe",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "windows 0.54.0",
+]
+
+[[package]]
+name = "dasp_sample"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"
+
+[[package]]
+name = "either"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
+
+[[package]]
+name = "equivalent"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
+
+[[package]]
+name = "find-msvc-tools"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
+
+[[package]]
+name = "futures-core"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+
+[[package]]
+name = "futures-task"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+
+[[package]]
+name = "futures-util"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "pin-project-lite",
+ "slab",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "r-efi",
+ "wasip2",
+]
+
+[[package]]
+name = "glob"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+
+[[package]]
+name = "hashbrown"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
+
+[[package]]
+name = "indexmap"
+version = "2.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
+dependencies = [
+ "equivalent",
+ "hashbrown",
+]
+
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "jni"
+version = "0.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"
+dependencies = [
+ "cesu8",
+ "cfg-if",
+ "combine",
+ "jni-sys 0.3.1",
+ "log",
+ "thiserror",
+ "walkdir",
+ "windows-sys 0.45.0",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
+dependencies = [
+ "jni-sys 0.4.1",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
+dependencies = [
+ "jni-sys-macros",
+]
+
+[[package]]
+name = "jni-sys-macros"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
+dependencies = [
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "jobserver"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
+dependencies = [
+ "getrandom",
+ "libc",
+]
+
+[[package]]
+name = "js-sys"
+version = "0.3.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "once_cell",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "jukebox-app"
+version = "0.1.0"
+dependencies = [
+ "rodio",
+ "windows 0.58.0",
+]
+
+[[package]]
+name = "jukebox-orchestrator"
+version = "0.1.0"
+dependencies = [
+ "midly",
+ "windows 0.58.0",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.186"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+
+[[package]]
+name = "libloading"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
+dependencies = [
+ "cfg-if",
+ "windows-link",
+]
+
+[[package]]
+name = "log"
+version = "0.4.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"
+
+[[package]]
+name = "mach2"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "memchr"
+version = "2.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
+
+[[package]]
+name = "midly"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "207d755f4cb882d20c4da58d707ca9130a0c9bc5061f657a4f299b8e36362b7a"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "ndk"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7"
+dependencies = [
+ "bitflags 2.11.1",
+ "jni-sys 0.3.1",
+ "log",
+ "ndk-sys",
+ "num_enum",
+ "thiserror",
+]
+
+[[package]]
+name = "ndk-context"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"
+
+[[package]]
+name = "ndk-sys"
+version = "0.5.0+25.2.9519653"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691"
+dependencies = [
+ "jni-sys 0.3.1",
+]
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "num-derive"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "num_enum"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
+dependencies = [
+ "num_enum_derive",
+ "rustversion",
+]
+
+[[package]]
+name = "num_enum_derive"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "oboe"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb"
+dependencies = [
+ "jni",
+ "ndk",
+ "ndk-context",
+ "num-derive",
+ "num-traits",
+ "oboe-sys",
+]
+
+[[package]]
+name = "oboe-sys"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.21.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
+
+[[package]]
+name = "proc-macro-crate"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
+dependencies = [
+ "toml_edit",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "r-efi"
+version = "5.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+
+[[package]]
+name = "regex"
+version = "1.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.8.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+
+[[package]]
+name = "rodio"
+version = "0.17.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b1bb7b48ee48471f55da122c0044fcc7600cfcc85db88240b89cb832935e611"
+dependencies = [
+ "cpal",
+]
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+
+[[package]]
+name = "rustversion"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.228"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
+[[package]]
+name = "slab"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
+
+[[package]]
+name = "syn"
+version = "2.0.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "1.1.1+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.25.12+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "toml_parser",
+ "winnow",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.1.2+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
+dependencies = [
+ "winnow",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "walkdir"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
+[[package]]
+name = "wasip2"
+version = "1.0.3+wasi-0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
+dependencies = [
+ "wit-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.122"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "rustversion",
+ "wasm-bindgen-macro",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.72"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.122"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.122"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
+dependencies = [
+ "bumpalo",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.122"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "web-sys"
+version = "0.3.99"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "winapi-util"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "windows"
+version = "0.54.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49"
+dependencies = [
+ "windows-core 0.54.0",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"
+dependencies = [
+ "windows-core 0.58.0",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.54.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65"
+dependencies = [
+ "windows-result 0.1.2",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-result 0.2.0",
+ "windows-strings",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-result"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
+dependencies = [
+ "windows-result 0.2.0",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets 0.42.2",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+dependencies = [
+ "windows_aarch64_gnullvm 0.42.2",
+ "windows_aarch64_msvc 0.42.2",
+ "windows_i686_gnu 0.42.2",
+ "windows_i686_msvc 0.42.2",
+ "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnullvm 0.42.2",
+ "windows_x86_64_msvc 0.42.2",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "winnow"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
diff --git a/demo/jukebox/Cargo.toml b/demo/jukebox/Cargo.toml
new file mode 100644
index 0000000000..9408253c7e
--- /dev/null
+++ b/demo/jukebox/Cargo.toml
@@ -0,0 +1,6 @@
+[workspace]
+resolver = "2"
+members = ["app", "orchestrator"]
+
+[profile.release]
+opt-level = 2
diff --git a/demo/jukebox/README.md b/demo/jukebox/README.md
new file mode 100644
index 0000000000..1d29461738
--- /dev/null
+++ b/demo/jukebox/README.md
@@ -0,0 +1,190 @@
+# CUA Jukebox β€” coordinated multi-cursor computer-use, as music
+
+A MIDI file (or a built-in demo song) becomes a **grid of miniwob-style
+minigame windows** β€” one per track. Each window gets its **own cua-driver
+session = its own uniquely-coloured agent cursor**. While the song plays, every
+note steers that track's cursor onto its widget and **clicks it in the
+background** β€” and the click is what makes the sound. One cursor per part, one
+colour per agent, all driven off a single clock: the dumbest possible orchestra,
+performed entirely by background computer-use.
+
+It's the native-windows sibling of the multi-cursor "National Records System"
+demo: same no-z-raise, no-real-mouse-movement background actuation, but here the
+*timing* is the point.
+
+```
+β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
+β”‚ CUA JUKEBOX β€” Transport β–Ά PLAY β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–‘β–‘β–‘β–‘β–‘β–‘β–‘β–‘ β”‚ ← you click PLAY
+β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€
+β”‚ Bass β—£crimsonβ”‚ Kick β—£amber β”‚ Hat β—£aqua β”‚ Pad β—£mint β”‚ … β”‚
+β”‚ [pitch strip] β”‚ [ KICK pad ] β”‚ [ HAT pad ] β”‚ [pitch strip]β”‚ β”‚ ← one agent
+β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ cursor each
+β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”˜
+```
+
+## What each window is
+
+| Window | Role | Widget | cursor actuation |
+|---|---|---|---|
+| Transport (controller) | foreground; the one human action | real Win32 `PLAY/STOP` button + playhead | you click it |
+| Kick / Snare / Hat | drum pad | one big pad; any click = a fixed hit | background click β†’ drum voice |
+| Bass / Lead / Pad / Arp / … | melodic | a **pitch strip**: the click's X selects the semitone | background click β†’ that pitch |
+
+Each instrument owns its **own `rodio` output stream**, so notes from the
+separate instrument processes mix at the OS mixer β€” genuine polyphony across the
+whole fleet. The track's name picks its minigame + synth voice (kick/snare/hat β†’
+pad; bass/lead/pad/arp/… β†’ pitch strip), mirroring the original HTML jukebox's
+`inferKind`.
+
+**Visual feedback:** a melodic strip lights the struck **key**, and each press
+fades on its own clock β€” so a chord lights several keys at once, individually. A
+drum pad instead has a single **brightness that gets an impulse per hit and
+constantly fades**, so the faster it's triggered the brighter it glows.
+
+## Cursor pools (one colour, many hands)
+
+A track normally has one cursor, but a cursor is "busy" for the whole click β€”
+the glide **plus** dispatch (β‰ˆ glide + 130 ms). When a track's notes fall closer
+together than that β€” a **chord** (simultaneous), or just a line faster than one
+cursor can service at the current glide β€” the first cursor is still busy, so the
+next free cursor in the track's **pool** takes the note, and the pool grows on
+demand (try the first, else the next, else spawn one; capped at 6). Every pool
+member is forced to the **same colour**, so a triad fans out into three
+identically-coloured cursors stabbing three keys at once, and a fast hi-hat line
+splits across two. Sizing the pool to the *real* click duration (not just the
+glide) is what keeps every cursor on the beat β€” otherwise one cursor would fall
+progressively behind on a track whose notes outpace its glide. The built-in demo
+plays Pad triads and dense hats/arps to show this. Slow / sparse tracks keep one
+cursor. Each pool cursor is its own cua-driver session + its own persistent
+connection + its own thread, so they actuate concurrently.
+
+## How the timing stays on the beat
+
+A background click only makes its sound once the cursor has *glided onto* the
+widget and tapped it, so the actuation lags the dispatch. Three things make it
+land on the beat anyway:
+
+1. **Fixed-duration glide.** Every cursor is pinned to a known, constant flight
+ time via `set_agent_cursor_motion {"glide_duration_ms": 200}` β€” so a 3-pixel
+ nudge and a cross-strip leap both arrive in the same 200 ms, making the
+ latency *predictable enough to sequence*.
+
+ > `glide_duration_ms` is honoured identically on macOS, Windows, and Linux β€”
+ > it lives in the shared cursor-overlay render core (`tick_motion` /
+ > `tick_swift_constants`). `0` (the default) keeps the original speed-based
+ > glide; any value `50–5000` forces that fixed flight time. No platform drift.
+
+2. **Persistent daemon connection.** The orchestrator holds **one named-pipe
+ connection per voice** to `cua-driver serve` and pipelines every click over
+ it β€” no `cua-driver call` *process spawn per note*. That spawn (tens of ms,
+ wildly variable) was the original timing-jitter source; removing it dropped
+ the per-note jitter from **sd β‰ˆ 490 ms β†’ β‰ˆ 35 ms**.
+
+3. **Per-voice adaptive lead + throughput-sized pools + 1 ms timer.** Each click
+ is fired early by an adaptive lead (a per-voice EMA tuned from the measured
+ error, warm-started at the dispatch overhead) so the mean error β†’ 0 without
+ one congested track skewing another. Pools are sized to the real click
+ duration (see above) so no single cursor outruns its glide. The system timer
+ is raised to 1 ms so `thread::sleep` schedules each click precisely. **And the
+ daemon must be built `--release`** β€” the overlay's per-frame pixel pipeline is
+ ~5Γ— slower in debug (β‰ˆ12 fps vs β‰ˆ60 fps under the full demo), and a slow
+ overlay = late, jittery glide-arrivals.
+
+The orchestrator **tracks the diff** itself: it records `(actual βˆ’ scheduled)`
+for every note and prints a report at song end, e.g.
+
+```
+[timing] n=256 mean=-4.0ms |mean|=19.4ms sd=35.8ms median=-0.8ms p10=-36 p90=+20 max=138
+```
+
+i.e. notes land on the beat to a sub-millisecond median with ~35 ms of jitter
+(about as tight as a human drummer), with a 200 ms glide. Tune with env vars:
+`JUKEBOX_GLIDE_MS` (default 200) and `JUKEBOX_LEAD_MS` (defaults to the glide,
+the initial per-voice pre-roll the adaptive correction refines from).
+
+## Build
+
+```powershell
+# from this directory
+cargo build
+# Build the driver RELEASE β€” the cursor overlay composites a full-virtual-screen
+# bitmap (RGBA→BGRA) and blits it every frame, which is ~5× slower unoptimized.
+# Debug: ~12 fps overlay under the full demo; release: ~60 fps (vsync-capped).
+cargo build -p cua-driver --release --manifest-path ..\..\libs\cua-driver\rust\Cargo.toml
+$env:CUA_DRIVER_EXE = "..\..\libs\cua-driver\rust\target\release\cua-driver.exe"
+```
+
+## Run
+
+```powershell
+.\target\debug\jukebox-orchestrator.exe # then click β–Ά PLAY in the Transport window
+.\target\debug\jukebox-orchestrator.exe --auto # self-starts after warmup
+.\target\debug\jukebox-orchestrator.exe song.mid # drive any multitrack .mid (best with named tracks)
+.\target\debug\jukebox-orchestrator.exe song.mid --auto
+```
+
+The orchestrator reaps any stale daemon, starts `cua-driver serve`, launches the
+Transport + one instrument window per track, tiles them (a 600Γ—80 bar over a grid
+of 200Γ—160 tiles), arms one coloured fixed-glide cursor pool per track, and on
+**PLAY** fans beat-synced background clicks out to every instrument. Press **Esc**
+on the Transport (or Ctrl-C the orchestrator) to tear everything down β€” a Windows
+Job Object kills the whole tree.
+
+## Bring your own song (MIDI)
+
+- **Drag a `.mid` onto `jukebox-orchestrator.exe`** in Explorer β€” Windows passes
+ it as the argument, so the file becomes the song.
+- …or pass it on the command line: `jukebox-orchestrator.exe path\to\song.mid`.
+
+It works best with **multitrack files that have named tracks** β€” the visualizer
+reads each track's name to pick its minigame + voice (`kick`/`snare`/`hat` β†’ drum
+pad; `bass`/`lead`/`pad`/`arp`/`string`/… β†’ pitch strip), and an unnamed track
+falls back to a sine pitch-strip. General-MIDI pop/electronic arrangements (one
+instrument per track, a drum track, a bass, a couple of leads/pads) map cleanly;
+chords on a track fan out into that track's same-colour cursor pool.
+
+Where to find MIDIs (always check each file's own licence):
+- **Open / Creative-Commons** score libraries β€” [Mutopia Project](https://www.mutopiaproject.org/),
+ [kunstderfuge](https://creativecommons.org/2008/03/07/kunstderfuge/) (CC BY-NC-SA),
+ the [Classical Piano MIDI Page](http://piano-midi.de/copy.htm) (CC BY-SA) β€” these
+ skew classical but are cleanly licensed and well-separated into tracks.
+- **CC audio search**: [Openverse](https://openverse.org/) and the
+ [Free Music Archive](https://freemusicarchive.org/curator/Creative_Commons/).
+- **Large general archives** (free downloads; licensing varies per file, so use for
+ personal/demo use): BitMidi (`bitmidi.com`), MidiWorld (`midiworld.com`),
+ FreeMidi (`freemidi.org`) β€” good for finding multitrack electronic/pop tracks.
+
+No `.mid`? The built-in generated 8-bar electronic loop (the default) is tuned to
+exercise every part of the visualizer β€” drums, a bass line, an arp, and Pad
+triads that show the cursor pool.
+
+### Env overrides
+`CUA_DRIVER_EXE`, `JUKEBOX_APP_EXE`, `JUKEBOX_GLIDE_MS` (default 200),
+`JUKEBOX_LEAD_MS` (defaults to the glide). Set
+`CUA_DRIVER_RS_OVERLAY_FPS_FILE=<path>` (read by the daemon) to log the agent
+cursor overlay's measured render FPS once a second.
+
+## How the coloured cursors work
+
+Each track's session is a **cua-driver palette name** (`crimson`, `amber`,
+`aqua`, `mint_lime`, `orchid`, …), so its overlay cursor renders in that palette
+automatically (`Palette::for_instance(session)`) β€” the same trick the
+multi-cursor demo uses. The instrument window's accent and the Transport legend
+reuse that palette's colour, so the cursor, its window, and the legend all read
+as one colour. (`cursor_color` on `set_agent_cursor_motion` only records a value;
+it doesn't repaint the overlay β€” true on macOS and Windows alike β€” so we key by
+palette name instead.)
+
+## Honest caveats
+
+- **Timing is groove-tight (median ~0 ms, jitter ~30 ms), not sample-accurate.**
+ The residual jitter is the overlay render tick (~8 ms) + OS scheduling. Under
+ heavy system load the daemon's single overlay render thread can starve, which
+ shows up as occasional multi-hundred-ms outliers on dense tracks; on an idle
+ machine all notes land in the Β±30 ms band.
+- **MIDI parsing uses a single tempo** (first tempo event wins) and ignores
+ channel/program data β€” instrument inference leans on track names. Untitled
+ tracks default to a sine pitch-strip.
+- Up to **9 tracks** (one per cua-driver palette); extra tracks are dropped.
+- Audio needs a default output device; with none, instruments still flash
+ silently.
diff --git a/demo/jukebox/app/Cargo.toml b/demo/jukebox/app/Cargo.toml
new file mode 100644
index 0000000000..5e92d2721f
--- /dev/null
+++ b/demo/jukebox/app/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "jukebox-app"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "jukebox-app"
+path = "src/main.rs"
+
+[dependencies]
+# Each instrument window owns its own audio output, so concurrent notes from
+# separate processes mix at the OS mixer β€” genuine multi-process polyphony.
+rodio = { version = "0.17", default-features = false }
+windows = { version = "0.58", features = [
+ "Win32_Foundation",
+ "Win32_UI_WindowsAndMessaging",
+ "Win32_UI_Shell",
+ "Win32_Graphics_Gdi",
+ "Win32_System_LibraryLoader",
+] }
diff --git a/demo/jukebox/app/src/main.rs b/demo/jukebox/app/src/main.rs
new file mode 100644
index 0000000000..c3d149d6d5
--- /dev/null
+++ b/demo/jukebox/app/src/main.rs
@@ -0,0 +1,604 @@
+//! "CUA JUKEBOX" β€” one window per part of a song, each a tiny miniwob-style
+//! minigame that an agent cursor actuates in time with the music. This binary
+//! is BOTH window kinds the demo launches:
+//!
+//! controller β€” the transport: a big PLAY/STOP button (a real Win32 control
+//! the human clicks), a track list, and a playhead. Emits
+//! `PLAY` / `STOP` on stdout for the orchestrator to react to.
+//!
+//! instrument β€” one minigame + its own synth voice. When something clicks
+//! the widget (the orchestrator drives these via cua-driver in
+//! the background), the window plays its note and flashes. Two
+//! widget kinds:
+//! pad β€” a single drum pad; any click = a fixed hit.
+//! keys β€” a pitch strip; the click's X selects the semitone,
+//! so the orchestrator "plays a melody" by choosing
+//! where on the strip each note lands.
+//!
+//! Every instrument owns its own `rodio` output stream, so notes from the
+//! separate instrument processes mix at the OS audio mixer β€” real polyphony
+//! across the whole coordinated fleet.
+
+#![windows_subsystem = "windows"]
+
+use std::cell::RefCell;
+use std::f32::consts::PI;
+use std::io::Write;
+use std::time::Instant;
+
+use windows::core::{w, PCWSTR};
+use windows::Win32::Foundation::{COLORREF, HINSTANCE, HWND, LPARAM, LRESULT, RECT, WPARAM};
+use windows::Win32::Graphics::Gdi::{
+ BeginPaint, BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, CreateFontW, CreateSolidBrush,
+ DeleteDC, DeleteObject, DrawTextW, EndPaint, FillRect, FrameRect, InvalidateRect, SelectObject,
+ SetBkMode, SetTextColor, DT_CENTER, DT_LEFT, DT_SINGLELINE, DT_VCENTER, HBRUSH, HFONT,
+ PAINTSTRUCT, SRCCOPY, TRANSPARENT,
+};
+use windows::Win32::System::LibraryLoader::GetModuleHandleW;
+use windows::Win32::UI::Shell::{DragAcceptFiles, DragFinish, DragQueryFileW, HDROP};
+use windows::Win32::UI::WindowsAndMessaging::*;
+
+// ── widget geometry (client fractions) β€” MUST match orchestrator's targets ────
+const WX0: f64 = 0.06; // widget left
+const WX1: f64 = 0.94; // widget right
+const WY0: f64 = 0.34; // widget top
+const WY1: f64 = 0.92; // widget bottom
+const KEYS_SPAN: i32 = 24; // semitones across a `keys` strip (2 octaves)
+
+// ── synth ─────────────────────────────────────────────────────────────────────
+#[derive(Clone, Copy, PartialEq)]
+enum Wave { Sine, Square, Saw, Triangle, Kick, Snare, Hat }
+
+impl Wave {
+ fn parse(s: &str) -> Wave {
+ match s {
+ "sine" => Wave::Sine,
+ "square" => Wave::Square,
+ "saw" => Wave::Saw,
+ "triangle" => Wave::Triangle,
+ "kick" => Wave::Kick,
+ "snare" => Wave::Snare,
+ "hat" => Wave::Hat,
+ _ => Wave::Sine,
+ }
+ }
+}
+
+fn midi_to_freq(m: f32) -> f32 { 440.0 * 2f32.powf((m - 69.0) / 12.0) }
+
+/// A short enveloped oscillator. One per note; rodio mixes overlapping ones.
+struct Tone {
+ sr: u32,
+ idx: u32,
+ total: u32,
+ phase: f32,
+ wave: Wave,
+ amp: f32,
+ f0: f32, // start frequency
+ f1: f32, // end frequency (0 = no pitch sweep)
+ rng: u32, // xorshift noise state
+ prev_noise: f32,
+}
+
+impl Tone {
+ fn note(wave: Wave, freq: f32, vel: f32) -> Tone {
+ let sr = 44_100u32;
+ let v = (vel / 127.0).clamp(0.05, 1.0);
+ let (dur, amp, f0, f1) = match wave {
+ Wave::Kick => (0.20, 0.55 * v, 155.0, 48.0),
+ Wave::Snare => (0.18, 0.34 * v, 180.0, 0.0),
+ Wave::Hat => (0.05, 0.24 * v, 9000.0, 0.0),
+ _ => (0.34, 0.17 * v, freq, 0.0),
+ };
+ Tone {
+ sr,
+ idx: 0,
+ total: (dur * sr as f32) as u32,
+ phase: 0.0,
+ wave,
+ amp,
+ f0,
+ f1,
+ rng: 0x9E37_79B9 ^ (freq as u32).wrapping_mul(2654435761),
+ prev_noise: 0.0,
+ }
+ }
+ fn noise(&mut self) -> f32 {
+ // xorshift32
+ let mut x = self.rng;
+ x ^= x << 13;
+ x ^= x >> 17;
+ x ^= x << 5;
+ self.rng = x;
+ (x as f32 / u32::MAX as f32) * 2.0 - 1.0
+ }
+}
+
+impl Iterator for Tone {
+ type Item = f32;
+ fn next(&mut self) -> Option<f32> {
+ if self.idx >= self.total { return None; }
+ let dur = self.total as f32 / self.sr as f32;
+ let t = self.idx as f32 / self.sr as f32;
+ let p = self.idx as f32 / self.total as f32;
+ let atk = 0.004;
+ let env = if t < atk { t / atk } else { (1.0 - (t - atk) / (dur - atk)).max(0.0).powf(1.6) };
+ let freq = if self.f1 > 0.0 { self.f0 * (self.f1 / self.f0).powf(p) } else { self.f0 };
+ self.phase += 2.0 * PI * freq / self.sr as f32;
+ if self.phase > 2.0 * PI { self.phase -= 2.0 * PI; }
+ let osc = match self.wave {
+ Wave::Sine | Wave::Kick => self.phase.sin(),
+ Wave::Square => if self.phase.sin() >= 0.0 { 1.0 } else { -1.0 },
+ Wave::Saw => self.phase / PI - 1.0,
+ Wave::Triangle => (2.0 / PI) * self.phase.sin().asin(),
+ Wave::Snare => { let n = self.noise(); 0.7 * n + 0.3 * self.phase.sin() }
+ Wave::Hat => { let n = self.noise(); let hp = n - self.prev_noise; self.prev_noise = n; hp }
+ };
+ self.idx += 1;
+ Some(osc * env * self.amp)
+ }
+}
+
+impl rodio::Source for Tone {
+ fn current_frame_len(&self) -> Option<usize> { None }
+ fn channels(&self) -> u16 { 1 }
+ fn sample_rate(&self) -> u32 { self.sr }
+ fn total_duration(&self) -> Option<std::time::Duration> { None }
+}
+
+// ── state ──────────────────────────────────────────────────────────────────────
+#[derive(Clone, Copy, PartialEq)]
+enum Mode { Controller, Instrument }
+
+#[derive(Clone, Copy, PartialEq)]
+enum Kind { Pad, Keys, Drums }
+
+/// One note actuation's fading highlight. `key` is the strip cell it lit (-1 for
+/// a drum pad); `inten` fades 1β†’0 and `age` grows so a pad press expands an
+/// outward ring. Each pulse fades on its own clock, so overlapping presses stay
+/// individually visible.
+#[derive(Clone, Copy)]
+struct Pulse { key: i32, inten: f32, age: f32 }
+
+struct Audio {
+ _stream: rodio::OutputStream,
+ handle: rodio::OutputStreamHandle,
+}
+
+struct State {
+ mode: Mode,
+ cw: i32,
+ ch: i32,
+ accent: COLORREF,
+ accent_rgb: (u8, u8, u8),
+ label: String,
+ f_title: HFONT,
+ f_label: HFONT,
+ f_big: HFONT,
+
+ // instrument
+ kind: Kind,
+ wave: Wave,
+ root: f32,
+ /// Semitones the pitch strip spans (fit to the track's range by the
+ /// orchestrator) β€” keys mapping + cell count use this, not the constant.
+ span: i32,
+ /// Melodic strips: recent note actuations, each fading on its own clock so
+ /// a chord fanned across the pool lights several keys at once, individually.
+ pulses: Vec<Pulse>,
+ /// Drum pads: a single brightness that constantly fades and gets an impulse
+ /// on each hit β€” so the faster you trigger it, the brighter it glows.
+ pad_glow: f32,
+ /// Drum kits: one such brightness per kick/snare/hat zone.
+ zone_glow: [f32; 3],
+ hits: u32,
+ audio: Option<Audio>,
+
+ // controller
+ bpm: u32,
+ dur_ms: u64,
+ tracks: Vec<(String, COLORREF)>,
+ hbtn: HWND,
+ playing: bool,
+ play_start: Option<Instant>,
+ /// Song time (ms) already elapsed in finished segments β€” frozen while paused
+ /// so the playhead (and the orchestrator's resume offset) hold position.
+ accum_ms: u64,
+}
+
+thread_local! { static STATE: RefCell<Option<State>> = const { RefCell::new(None) }; }
+
+const ID_PLAY: isize = 2001;
+// Single transport button is an icon: β–Ά when stopped/paused, ⏸ while playing.
+const ICON_PLAY: PCWSTR = w!("β–Ά");
+const ICON_PAUSE: PCWSTR = w!("⏸");
+
+fn emit(line: &str) { let _ = writeln!(std::io::stdout(), "{line}"); let _ = std::io::stdout().flush(); }
+fn wide(s: &str) -> Vec<u16> { s.encode_utf16().chain(std::iter::once(0)).collect() }
+fn arg(args: &[String], flag: &str) -> Option<String> {
+ args.iter().position(|a| a == flag).and_then(|i| args.get(i + 1).cloned())
+}
+fn parse_color(s: &str) -> (COLORREF, (u8, u8, u8)) {
+ let h = s.trim_start_matches('#');
+ let v = u32::from_str_radix(h, 16).unwrap_or(0x00FFFF);
+ let (r, g, b) = (((v >> 16) & 255) as u8, ((v >> 8) & 255) as u8, (v & 255) as u8);
+ (COLORREF(((b as u32) << 16) | ((g as u32) << 8) | r as u32), (r, g, b))
+}
+fn rgb(r: u8, g: u8, b: u8) -> COLORREF { COLORREF(((b as u32) << 16) | ((g as u32) << 8) | r as u32) }
+fn mix(a: (u8, u8, u8), b: (u8, u8, u8), t: f32) -> COLORREF {
+ let f = |x: u8, y: u8| (x as f32 + (y as f32 - x as f32) * t) as u8;
+ rgb(f(a.0, b.0), f(a.1, b.1), f(a.2, b.2))
+}
+fn fr(cw: i32, ch: i32, l: f64, t: f64, r: f64, b: f64) -> RECT {
+ RECT { left: (cw as f64 * l) as i32, top: (ch as f64 * t) as i32,
+ right: (cw as f64 * r) as i32, bottom: (ch as f64 * b) as i32 }
+}
+unsafe fn mk_font(h: i32, w: i32, face: PCWSTR) -> HFONT { CreateFontW(h, 0, 0, 0, w, 0, 0, 0, 0, 0, 0, 0, 0, face) }
+
+fn main() {
+ let args: Vec<String> = std::env::args().collect();
+ let mode = if args.get(1).map(|s| s.as_str()) == Some("controller") { Mode::Controller } else { Mode::Instrument };
+ let title = arg(&args, "--title").unwrap_or_else(|| "JUKEBOX".into());
+ let (accent, accent_rgb) = parse_color(&arg(&args, "--color").unwrap_or_else(|| "#3bd0ff".into()));
+ let label = arg(&args, "--label").unwrap_or_else(|| "PART".into());
+ let kind = match arg(&args, "--kind").as_deref() {
+ Some("keys") => Kind::Keys,
+ Some("drums") => Kind::Drums,
+ _ => Kind::Pad,
+ };
+ let wave = Wave::parse(&arg(&args, "--wave").unwrap_or_else(|| "sine".into()));
+ let root: f32 = arg(&args, "--root").and_then(|s| s.parse().ok()).unwrap_or(48.0);
+ let span: i32 = arg(&args, "--span").and_then(|s| s.parse().ok()).unwrap_or(KEYS_SPAN).clamp(1, 96);
+ let bpm: u32 = arg(&args, "--bpm").and_then(|s| s.parse().ok()).unwrap_or(120);
+ let dur_ms: u64 = arg(&args, "--dur-ms").and_then(|s| s.parse().ok()).unwrap_or(16000);
+ let tracks: Vec<(String, COLORREF)> = arg(&args, "--tracks").map(|s| s.split(',')
+ .filter_map(|t| { let mut it = t.splitn(2, '|'); Some((it.next()?.to_string(), parse_color(it.next().unwrap_or("#888")).0)) })
+ .collect()).unwrap_or_default();
+
+ let audio = if mode == Mode::Instrument {
+ rodio::OutputStream::try_default().ok().map(|(s, h)| Audio { _stream: s, handle: h })
+ } else { None };
+
+ unsafe {
+ let hmod = GetModuleHandleW(None).unwrap();
+ let class = w!("CuaJukeboxWindow");
+ let bg = CreateSolidBrush(rgb(0x0b, 0x0c, 0x12));
+ let wc = WNDCLASSW { lpfnWndProc: Some(wnd_proc), hInstance: hmod.into(), lpszClassName: class,
+ hbrBackground: bg, hCursor: LoadCursorW(None, IDC_ARROW).unwrap_or_default(), ..Default::default() };
+ RegisterClassW(&wc);
+
+ STATE.with(|s| *s.borrow_mut() = Some(State {
+ mode, cw: 0, ch: 0, accent, accent_rgb, label,
+ f_title: HFONT::default(), f_label: HFONT::default(), f_big: HFONT::default(),
+ kind, wave, root, span, pulses: Vec::new(), pad_glow: 0.0, zone_glow: [0.0; 3], hits: 0, audio,
+ bpm, dur_ms, tracks, hbtn: HWND::default(),
+ playing: false, play_start: None, accum_ms: 0,
+ }));
+
+ let tw = wide(&title);
+ // Fixed sizes the orchestrator also lays out against: a thin 600Γ—80
+ // transport bar, and tight 200Γ—160 instrument tiles. Borderless
+ // (WS_POPUP) so the tiles pack together with no caption/frame β€” the
+ // whole client is the visualizer. (Esc on the focused transport quits.)
+ let (dw, dh) = if mode == Mode::Controller { (600, 80) } else { (200, 160) };
+ let hwnd = CreateWindowExW(WINDOW_EX_STYLE(0), class, PCWSTR(tw.as_ptr()), WS_POPUP | WS_VISIBLE,
+ CW_USEDEFAULT, CW_USEDEFAULT, dw, dh, None, None, HINSTANCE(hmod.0), None).expect("CreateWindowExW");
+ let _ = ShowWindow(hwnd, SW_SHOWNORMAL);
+ SetTimer(hwnd, 1, 40, None); // ~25fps: glow fade / playhead (region-clipped)
+
+ let mut msg = MSG::default();
+ while GetMessageW(&mut msg, None, 0, 0).as_bool() { let _ = TranslateMessage(&msg); DispatchMessageW(&msg); }
+ }
+}
+
+unsafe fn relayout(hwnd: HWND, cw: i32, ch: i32) {
+ STATE.with(|s| {
+ let mut b = s.borrow_mut(); let Some(st) = b.as_mut() else { return };
+ st.cw = cw; st.ch = ch;
+ for f in [st.f_title, st.f_label, st.f_big] { if !f.is_invalid() { let _ = DeleteObject(f); } }
+ let seg = wide("Segoe UI"); let mono = wide("Consolas");
+ st.f_title = mk_font(-(ch / 16).clamp(14, 30), 800, PCWSTR(seg.as_ptr()));
+ st.f_label = mk_font(-(ch / 26).clamp(11, 18), 500, PCWSTR(mono.as_ptr()));
+ st.f_big = mk_font(-(ch / 9).clamp(20, 64), 800, PCWSTR(seg.as_ptr()));
+ if st.mode == Mode::Controller && !st.hbtn.0.is_null() {
+ // Thin transport bar: one small square icon button at the left edge,
+ // leaving room for the title. Big font so the β–Ά / ⏸ glyph reads.
+ let r = fr(cw, ch, 0.012, 0.16, 0.085, 0.84);
+ let _ = MoveWindow(st.hbtn, r.left, r.top, r.right - r.left, r.bottom - r.top, true);
+ SendMessageW(st.hbtn, WM_SETFONT, WPARAM(st.f_big.0 as usize), LPARAM(1));
+ }
+ });
+ let _ = InvalidateRect(hwnd, None, true);
+}
+
+extern "system" fn wnd_proc(hwnd: HWND, msg: u32, wp: WPARAM, lp: LPARAM) -> LRESULT {
+ unsafe {
+ match msg {
+ WM_CREATE => {
+ if STATE.with(|s| s.borrow().as_ref().map(|st| st.mode)) == Some(Mode::Controller) {
+ let hinst = HINSTANCE(GetModuleHandleW(None).unwrap().0);
+ let hbtn = CreateWindowExW(
+ WINDOW_EX_STYLE(0), w!("BUTTON"), ICON_PLAY,
+ WS_CHILD | WS_VISIBLE | WINDOW_STYLE((BS_PUSHBUTTON | BS_CENTER | BS_VCENTER) as u32),
+ 0, 0, 10, 10, hwnd, HMENU(ID_PLAY as *mut core::ffi::c_void), hinst, None).unwrap_or_default();
+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() { st.hbtn = hbtn; });
+ DragAcceptFiles(hwnd, true); // drop a .mid on the transport to load it
+ }
+ LRESULT(0)
+ }
+ WM_DROPFILES => {
+ let hdrop = HDROP(wp.0 as *mut core::ffi::c_void);
+ let mut buf = [0u16; 1024];
+ let n = DragQueryFileW(hdrop, 0, Some(&mut buf));
+ if n > 0 {
+ let path = String::from_utf16_lossy(&buf[..n as usize]);
+ emit(&format!("LOAD\t{path}")); // orchestrator restarts with this track
+ }
+ DragFinish(hdrop);
+ LRESULT(0)
+ }
+ WM_SIZE => { let (cw, ch) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32); if cw > 0 && ch > 0 { relayout(hwnd, cw, ch); } LRESULT(0) }
+ WM_COMMAND => {
+ let (id, code) = ((wp.0 & 0xFFFF) as isize, ((wp.0 >> 16) & 0xFFFF) as u32);
+ if code == BN_CLICKED && id == ID_PLAY { on_play_toggle(hwnd); }
+ LRESULT(0)
+ }
+ WM_LBUTTONDOWN => {
+ if STATE.with(|s| s.borrow().as_ref().map(|st| st.mode)) == Some(Mode::Instrument) {
+ let x = (lp.0 & 0xFFFF) as i16 as i32;
+ actuate(hwnd, x);
+ }
+ LRESULT(0)
+ }
+ WM_TIMER => { on_tick(hwnd); LRESULT(0) }
+ // Esc quits (borderless windows have no close button); closing the
+ // transport EOFs its stdout, which tears the whole demo down.
+ WM_KEYDOWN if wp.0 == 0x1B => { PostQuitMessage(0); LRESULT(0) }
+ // We fully repaint every frame via a double-buffered WM_PAINT, so
+ // suppress the default background erase β€” that erase-then-paint is
+ // the other half of GDI flicker.
+ WM_ERASEBKGND => LRESULT(1),
+ WM_PAINT => { paint(hwnd); LRESULT(0) }
+ WM_DESTROY => { PostQuitMessage(0); LRESULT(0) }
+ _ => DefWindowProcW(hwnd, msg, wp, lp),
+ }
+ }
+}
+
+/// Instrument actuation: the click's X selects what to play (pitch on a strip,
+/// or which kick/snare/hat zone on a kit), then play the voice + flash. Fires
+/// whether the click came from the human or β€” the point β€” from cua-driver.
+unsafe fn actuate(hwnd: HWND, x: i32) {
+ STATE.with(|s| {
+ let mut b = s.borrow_mut(); let Some(st) = b.as_mut() else { return };
+ let cw = st.cw.max(1) as f64;
+ let frac = ((x as f64 / cw) - WX0) / (WX1 - WX0);
+ let (key, freq, wave, vel) = match st.kind {
+ Kind::Keys => {
+ let key = (frac * st.span as f64).floor().clamp(0.0, (st.span - 1) as f64) as i32;
+ // Each press is its OWN pulse, fading independently β€” a chord
+ // lights several keys at once, each on its own clock.
+ st.pulses.push(Pulse { key, inten: 1.0, age: 0.0 });
+ if st.pulses.len() > 32 { st.pulses.remove(0); }
+ (key, midi_to_freq(st.root + key as f32), st.wave, 104.0)
+ }
+ Kind::Drums => {
+ // X selects the kick/snare/hat zone; light that zone's brightness.
+ let z = (frac * 3.0).floor().clamp(0.0, 2.0) as usize;
+ st.zone_glow[z] = (st.zone_glow[z] + 0.5).min(1.0);
+ let w = [Wave::Kick, Wave::Snare, Wave::Hat][z];
+ (z as i32, 0.0, w, 112.0)
+ }
+ Kind::Pad => {
+ // Impulse on a constantly-fading brightness: rapid hits
+ // accumulate toward full, sparse hits fade between them.
+ st.pad_glow = (st.pad_glow + 0.5).min(1.0);
+ (-1, midi_to_freq(st.root), st.wave, 112.0)
+ }
+ };
+ st.hits += 1;
+ if let Some(a) = &st.audio { let _ = a.handle.play_raw(Tone::note(wave, freq, vel)); }
+ // Heartbeat for the orchestrator / verification (stdout is null in
+ // normal runs, so this is a no-op there).
+ emit(&format!("HIT {} key={} {:.0}Hz", st.hits, key, freq));
+ });
+ let _ = InvalidateRect(hwnd, None, false);
+}
+
+/// The single transport button is a play/pause toggle: playing β†’ pause (freeze
+/// at the current position so the next press resumes from here); paused/stopped
+/// β†’ play/resume. The icon flips β–Ά ⇄ ⏸.
+unsafe fn on_play_toggle(hwnd: HWND) {
+ STATE.with(|s| {
+ let mut b = s.borrow_mut(); let Some(st) = b.as_mut() else { return };
+ if st.playing {
+ // β†’ pause: freeze position, show the β–Ά (play/resume) icon.
+ if let Some(t0) = st.play_start { st.accum_ms += t0.elapsed().as_millis() as u64; }
+ st.playing = false; st.play_start = None;
+ emit("PAUSE"); let _ = SetWindowTextW(st.hbtn, ICON_PLAY);
+ } else {
+ // β†’ play/resume: show the ⏸ (pause) icon.
+ st.playing = true; st.play_start = Some(Instant::now());
+ emit("PLAY"); let _ = SetWindowTextW(st.hbtn, ICON_PAUSE);
+ }
+ });
+ let _ = InvalidateRect(hwnd, None, true);
+}
+
+/// Current song position in ms (accumulated finished segments + the running one).
+fn position_ms(st: &State) -> u64 {
+ st.accum_ms + st.play_start.map(|t| t.elapsed().as_millis() as u64).unwrap_or(0)
+}
+
+unsafe fn on_tick(hwnd: HWND) {
+ // Returns the region to repaint this tick (None = nothing changed). Only the
+ // changing band is invalidated, so an idle header/legend never re-composites
+ // β€” that frees DWM/GPU bandwidth for the agent-cursor overlay (which is the
+ // expensive full-virtual-screen layered window the cursors live on).
+ let region: Option<RECT> = STATE.with(|s| {
+ let mut b = s.borrow_mut(); let st = b.as_mut()?;
+ let (cw, ch) = (st.cw.max(1), st.ch.max(1));
+ match st.mode {
+ Mode::Instrument => {
+ let had = !st.pulses.is_empty() || st.pad_glow > 0.01
+ || st.zone_glow.iter().any(|&z| z > 0.01);
+ for p in st.pulses.iter_mut() { p.age += 0.040; p.inten *= 0.88; }
+ st.pulses.retain(|p| p.inten > 0.03);
+ st.pad_glow *= 0.88;
+ if st.pad_glow < 0.01 { st.pad_glow = 0.0; }
+ for z in st.zone_glow.iter_mut() { *z *= 0.88; if *z < 0.01 { *z = 0.0; } }
+ had.then(|| fr(cw, ch, 0.0, WY0 - 0.03, 1.0, 1.0)) // widget band only
+ }
+ Mode::Controller => {
+ if !st.playing { return None; }
+ if position_ms(st) > st.dur_ms + 250 {
+ // Song ended β†’ reset to the top. Emit STOP so the orchestrator
+ // clears its playing flag + resets its resume offset to 0.
+ st.playing = false; st.play_start = None; st.accum_ms = 0;
+ emit("STOP"); let _ = SetWindowTextW(st.hbtn, ICON_PLAY);
+ return Some(fr(cw, ch, 0.0, 0.0, 1.0, 1.0)); // full repaint once
+ }
+ Some(fr(cw, ch, 0.0, 0.91, 1.0, 1.0)) // playhead bar only
+ }
+ }
+ });
+ if let Some(r) = region { let _ = InvalidateRect(hwnd, Some(&r), false); }
+}
+
+unsafe fn text(hdc: windows::Win32::Graphics::Gdi::HDC, r: RECT, s: &str, fmt: windows::Win32::Graphics::Gdi::DRAW_TEXT_FORMAT) {
+ let mut t = wide(s); let mut rr = r;
+ DrawTextW(hdc, &mut t, &mut rr, fmt | DT_SINGLELINE);
+}
+
+unsafe fn paint(hwnd: HWND) {
+ let mut ps = PAINTSTRUCT::default();
+ let hdc = BeginPaint(hwnd, &mut ps);
+ STATE.with(|s| {
+ let b = s.borrow(); let Some(st) = b.as_ref() else { return };
+ let (cw, ch) = (st.cw.max(1), st.ch.max(1));
+ // Double-buffer: build the whole frame in an off-screen DC, then blit it
+ // to the window in one BitBlt. Painting straight to the window DC (with a
+ // full-client background fill every frame, ~30fps from the glow/playhead
+ // timer) is what caused the flicker β€” never the windows being recreated.
+ let mem = CreateCompatibleDC(hdc);
+ let bmp = CreateCompatibleBitmap(hdc, cw, ch);
+ let old = SelectObject(mem, bmp);
+ SetBkMode(mem, TRANSPARENT);
+
+ let bg = CreateSolidBrush(rgb(0x0b, 0x0c, 0x12));
+ let panel = CreateSolidBrush(rgb(0x16, 0x18, 0x22));
+ let line = CreateSolidBrush(rgb(0x2a, 0x2d, 0x3e));
+ let dim = rgb(0x6a, 0x6f, 0x85);
+ let ink = rgb(0xe8, 0xea, 0xf2);
+ let full = RECT { left: 0, top: 0, right: cw, bottom: ch };
+ FillRect(mem, &full, bg);
+
+ match st.mode {
+ Mode::Controller => paint_controller(mem, st, cw, ch, &panel, &line, ink, dim),
+ Mode::Instrument => paint_instrument(mem, st, cw, ch, &panel, &line, ink, dim),
+ }
+ for o in [bg, panel, line] { let _ = DeleteObject(o); }
+
+ let _ = BitBlt(hdc, 0, 0, cw, ch, mem, 0, 0, SRCCOPY);
+ SelectObject(mem, old);
+ let _ = DeleteObject(bmp);
+ let _ = DeleteDC(mem);
+ });
+ let _ = EndPaint(hwnd, &ps);
+}
+
+unsafe fn paint_controller(hdc: windows::Win32::Graphics::Gdi::HDC, st: &State, cw: i32, ch: i32, _panel: &HBRUSH, line: &HBRUSH, _ink: COLORREF, dim: COLORREF) {
+ // Thin transport bar. A small icon play/pause button sits at the far left
+ // (~9%); lay the rest out horizontally: title, bpm, track swatches, playhead.
+ SelectObject(hdc, st.f_title); SetTextColor(hdc, rgb(0xff, 0xff, 0xff));
+ text(hdc, fr(cw, ch, 0.11, 0.06, 0.55, 0.58), "CUA JUKEBOX", DT_LEFT | DT_VCENTER);
+ SelectObject(hdc, st.f_label); SetTextColor(hdc, dim);
+ text(hdc, fr(cw, ch, 0.115, 0.52, 0.55, 0.95),
+ &format!("{} parts Β· {} bpm", st.tracks.len(), st.bpm), DT_LEFT | DT_VCENTER);
+
+ // Row of track colour swatches (matches each instrument tile's colour).
+ let n = st.tracks.len().max(1);
+ let (x0, x1) = (0.56_f64, 0.985_f64);
+ let sw_w = (x1 - x0) / n as f64;
+ for (i, (_name, col)) in st.tracks.iter().enumerate() {
+ let sx = x0 + i as f64 * sw_w;
+ let sw = fr(cw, ch, sx, 0.18, sx + sw_w * 0.7, 0.66);
+ let cb = CreateSolidBrush(*col); FillRect(hdc, &sw, cb); let _ = DeleteObject(cb);
+ }
+
+ // Playhead along the very bottom.
+ let bar = fr(cw, ch, 0.0, 0.92, 1.0, 1.0);
+ FillRect(hdc, &bar, *line);
+ let frac = (position_ms(st) as f64 / st.dur_ms.max(1) as f64).clamp(0.0, 1.0);
+ if frac > 0.0 {
+ let mut fb = bar; fb.right = bar.left + ((bar.right - bar.left) as f64 * frac) as i32;
+ // Dim while paused, bright while playing.
+ let c = if st.playing { st.accent } else { mix((0x2a, 0x2d, 0x3e), st.accent_rgb, 0.45) };
+ let cb = CreateSolidBrush(c); FillRect(hdc, &fb, cb); let _ = DeleteObject(cb);
+ }
+}
+
+unsafe fn paint_instrument(hdc: windows::Win32::Graphics::Gdi::HDC, st: &State, cw: i32, ch: i32, _panel: &HBRUSH, line: &HBRUSH, ink: COLORREF, dim: COLORREF) {
+ let black = (0x0b, 0x0c, 0x12);
+ // header: swatch + label + hits
+ let sw = fr(cw, ch, 0.06, 0.09, 0.10, 0.17);
+ let cb = CreateSolidBrush(st.accent); FillRect(hdc, &sw, cb); let _ = DeleteObject(cb);
+ SelectObject(hdc, st.f_title); SetTextColor(hdc, ink);
+ text(hdc, fr(cw, ch, 0.13, 0.07, 0.78, 0.20), &st.label, DT_LEFT | DT_VCENTER);
+ SelectObject(hdc, st.f_label); SetTextColor(hdc, dim);
+ text(hdc, fr(cw, ch, 0.60, 0.07, 0.95, 0.20), &format!("{}β™ͺ", st.hits), DT_LEFT | DT_VCENTER);
+
+ let frame = |r: &RECT, c: COLORREF| { let br = CreateSolidBrush(c); FrameRect(hdc, r, HBRUSH(br.0)); let _ = DeleteObject(br); };
+ let panel_bg = (0x20, 0x23, 0x30);
+
+ match st.kind {
+ Kind::Pad => {
+ // Brightness = the constantly-fading, impulse-driven pad_glow: fast
+ // hits pile up to a bright pad, sparse hits let it dim between them.
+ let g = st.pad_glow.clamp(0.0, 1.0);
+ let pad = fr(cw, ch, WX0 + 0.10, WY0, WX1 - 0.10, WY1);
+ let body = mix(panel_bg, st.accent_rgb, g * 0.9);
+ let bb = CreateSolidBrush(body); FillRect(hdc, &pad, bb); let _ = DeleteObject(bb);
+ frame(&pad, st.accent);
+ SelectObject(hdc, st.f_big);
+ SetTextColor(hdc, if g > 0.4 { rgb(black.0, black.1, black.2) } else { st.accent });
+ let cap = match st.wave { Wave::Kick => "KICK", Wave::Snare => "SNARE", Wave::Hat => "HAT", _ => "PAD" };
+ text(hdc, pad, cap, DT_CENTER | DT_VCENTER);
+ }
+ Kind::Keys => {
+ let strip = fr(cw, ch, WX0, WY0, WX1, WY1);
+ FillRect(hdc, &strip, *line);
+ let span = st.span.max(1);
+ let w = (strip.right - strip.left) as f64 / span as f64;
+ for k in 0..span {
+ let kx = strip.left + (k as f64 * w) as i32;
+ let cell = RECT { left: kx + 1, top: strip.top + 1, right: kx + w as i32 - 1, bottom: strip.bottom - 1 };
+ // Each cell glows by the brightest pulse on THAT key β€” so a chord
+ // lights several cells at once, each fading independently.
+ let inten = st.pulses.iter().filter(|p| p.key == k).map(|p| p.inten).fold(0.0_f32, f32::max);
+ let c = if inten > 0.02 { mix(panel_bg, st.accent_rgb, inten) }
+ else if k % 12 == 0 { rgb(0x20, 0x23, 0x30) } else { rgb(0x18, 0x1a, 0x24) };
+ let bb = CreateSolidBrush(c); FillRect(hdc, &cell, bb); let _ = DeleteObject(bb);
+ }
+ frame(&strip, st.accent);
+ SelectObject(hdc, st.f_label); SetTextColor(hdc, dim);
+ text(hdc, fr(cw, ch, WX0, WY1 + 0.005, WX1, 0.99), "pitch β—„ low Β· high β–Ί", DT_CENTER | DT_VCENTER);
+ }
+ Kind::Drums => {
+ // Three pads β€” KICK Β· SNARE Β· HAT β€” each its own impulse/fade glow.
+ let caps = ["KICK", "SNARE", "HAT"];
+ SelectObject(hdc, st.f_label);
+ for z in 0..3usize {
+ let zx0 = WX0 + z as f64 * (WX1 - WX0) / 3.0;
+ let zx1 = WX0 + (z as f64 + 1.0) * (WX1 - WX0) / 3.0;
+ let pad = fr(cw, ch, zx0 + 0.01, WY0, zx1 - 0.01, WY1);
+ let g = st.zone_glow[z].clamp(0.0, 1.0);
+ let body = mix(panel_bg, st.accent_rgb, g * 0.9);
+ let bb = CreateSolidBrush(body); FillRect(hdc, &pad, bb); let _ = DeleteObject(bb);
+ frame(&pad, st.accent);
+ SetTextColor(hdc, if g > 0.4 { rgb(black.0, black.1, black.2) } else { st.accent });
+ text(hdc, pad, caps[z], DT_CENTER | DT_VCENTER);
+ }
+ }
+ }
+}
diff --git a/demo/jukebox/orchestrator/Cargo.toml b/demo/jukebox/orchestrator/Cargo.toml
new file mode 100644
index 0000000000..913282c5cf
--- /dev/null
+++ b/demo/jukebox/orchestrator/Cargo.toml
@@ -0,0 +1,22 @@
+[package]
+name = "jukebox-orchestrator"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "jukebox-orchestrator"
+path = "src/main.rs"
+
+[dependencies]
+# Minimal, well-tested SMF parser so real multitrack .mid files Just Work.
+midly = { version = "0.5", default-features = false, features = ["std"] }
+windows = { version = "0.58", features = [
+ "Win32_Foundation",
+ "Win32_UI_WindowsAndMessaging",
+ "Win32_Graphics_Gdi",
+ "Win32_Graphics_Dwm",
+ "Win32_System_JobObjects",
+ "Win32_System_Threading",
+ "Win32_Security",
+ "Win32_Media",
+] }
diff --git a/demo/jukebox/orchestrator/src/main.rs b/demo/jukebox/orchestrator/src/main.rs
new file mode 100644
index 0000000000..d1d3e1f55a
--- /dev/null
+++ b/demo/jukebox/orchestrator/src/main.rs
@@ -0,0 +1,841 @@
+//! CUA JUKEBOX orchestrator β€” coordinated multi-cursor "computer-use" music.
+//!
+//! Reads a MIDI file (or a built-in demo song), turns each track into its own
+//! instrument window (a miniwob-style minigame), and gives each one its OWN
+//! cua-driver session = its OWN uniquely-coloured agent cursor. While the song
+//! plays, every note steers that track's cursor onto its widget and clicks it
+//! in the background β€” the click is what makes the sound. One cursor per part,
+//! one colour per agent, all driven off a single clock: the dumbest possible
+//! orchestra, performed entirely by background computer-use.
+//!
+//! Timing is locked with the cursor's `glide_duration_ms` (set per session via
+//! `set_agent_cursor_motion`): every glide takes a known, fixed time regardless
+//! of how far the cursor must travel across its pitch strip, so the orchestrator
+//! can pre-roll each click by exactly that lead and land the actuation on the
+//! beat. (That field is honoured identically on macOS and Windows β€” it lives in
+//! the shared render core.)
+
+use std::path::PathBuf;
+use std::process::{Child, Command, Stdio};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::Arc;
+use std::thread;
+use std::time::{Duration, Instant};
+
+use windows::Win32::Foundation::{BOOL, HANDLE, HWND, LPARAM, POINT, RECT, TRUE};
+use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS};
+use windows::Win32::Graphics::Gdi::ClientToScreen;
+use windows::Win32::System::JobObjects::{
+ AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject,
+ JobObjectExtendedLimitInformation, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
+ JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
+};
+use windows::Win32::UI::WindowsAndMessaging::{
+ EnumWindows, GetClientRect, GetWindowTextW, GetWindowThreadProcessId, IsWindowVisible,
+ SetForegroundWindow, SetWindowPos, HWND_TOP, SWP_NOACTIVATE, SWP_NOZORDER, SWP_SHOWWINDOW,
+};
+
+// Widget geometry β€” MUST match jukebox-app's WX0/WX1/WY0/WY1/KEYS_SPAN.
+const WX0: f64 = 0.06;
+const WX1: f64 = 0.94;
+const WY0: f64 = 0.34;
+const WY1: f64 = 0.92;
+const KEYS_SPAN: i32 = 24;
+const MAX_TRACKS: usize = 9;
+
+// One cua-driver palette per agent. Keying a session to a palette NAME makes
+// that session's overlay cursor render in that palette automatically
+// (`Palette::for_instance(session)`), exactly like the sibling multi-cursor
+// demo. The hex is that palette's mid colour, reused for the instrument window
+// accent AND the transport legend, so the cursor, its window, and the legend
+// all read as the same colour. (`cursor_color` on `set_agent_cursor_motion`
+// only updates registry state, not the Windows overlay paint β€” so we rely on
+// the session-name palette instead of fighting it.)
+const SESSIONS: [(&str, &str); 9] = [
+ ("crimson", "#e85262"),
+ ("amber", "#f4b242"),
+ ("aqua", "#4ccce0"),
+ ("mint_lime", "#60daae"),
+ ("orchid", "#dd71ec"),
+ ("soft_purple", "#b284ff"),
+ ("rose_gold", "#f784aa"),
+ ("chartreuse", "#b8dc36"),
+ ("cobalt", "#507eec"),
+];
+
+#[derive(Clone, Copy, PartialEq)]
+enum Kind { Pad, Keys, Drums }
+
+#[derive(Clone)]
+struct Note {
+ t: f64,
+ pitch: u8,
+ /// Carried from the source for fidelity; the instrument picks its own hit
+ /// velocity (a background click can't carry one), so it's unused here.
+ #[allow(dead_code)]
+ vel: u8,
+}
+
+struct Track {
+ name: String,
+ notes: Vec<Note>,
+ kind: Kind,
+ wave: &'static str,
+ root: i32,
+ /// Number of semitones the pitch strip spans (fit to the track's range so
+ /// wide melodies aren't clamped to the top key). Drums ignore it.
+ span: i32,
+ color: &'static str,
+ session: String,
+ title: String,
+ hwnd: HWND,
+ pid: u32,
+ /// True when this track is percussion (most notes on MIDI channel 10, or a
+ /// drum-kit name) β†’ rendered as a 3-zone kick/snare/hat pad.
+ is_drum: bool,
+}
+
+struct Song { tracks: Vec<Track>, bpm: u32, dur_sec: f64 }
+
+/// Send-able snapshot of one cursor's worth of work (a "voice" β€” a single pool
+/// member of a track). HWND is carried as an isize because the raw handle
+/// pointer isn't `Send`. Several voices can share one track's window/colour.
+struct Voice {
+ pid: u32,
+ hwnd_addr: isize,
+ session: String,
+ kind: Kind,
+ root: i32,
+ span: i32,
+ notes: Arc<Vec<Note>>,
+}
+
+/// Max same-colour cursors a single track may grow to (a chord wider than this
+/// reuses cursor 0 and accepts a little overlap rather than spawning forever).
+const MAX_POOL: usize = 6;
+
+/// Assign each (time-sorted) note to a pool cursor: try the lowest-indexed
+/// cursor that's free (its previous glide finished β‰₯ now), else grow the pool,
+/// else (at the cap) reuse cursor 0. `busy` is the glide window a cursor is
+/// "occupied" for after it fires. Returns the per-note cursor index and the
+/// resulting pool size. This is the deterministic form of "try the first, go to
+/// the next if occupied during the 100ms, spawn one if none free".
+fn assign_pool(notes: &[Note], busy: f64, cap: usize) -> (Vec<usize>, usize) {
+ let mut free_at: Vec<f64> = Vec::new();
+ let mut assign = Vec::with_capacity(notes.len());
+ for n in notes {
+ let chosen = free_at.iter().position(|&f| f <= n.t + 1e-6);
+ let i = match chosen {
+ Some(i) => i,
+ None if free_at.len() < cap => { free_at.push(0.0); free_at.len() - 1 }
+ None => 0,
+ };
+ free_at[i] = n.t + busy;
+ assign.push(i);
+ }
+ (assign, free_at.len().max(1))
+}
+
+/// Blend a `#rrggbb` toward white by `t` (0=unchanged, 1=white) for a gradient
+/// tip stop, so a forced-colour cursor still reads as a lit arrow, not a flat blob.
+fn lighten(hex: &str, t: f64) -> String {
+ let v = u32::from_str_radix(hex.trim_start_matches('#'), 16).unwrap_or(0xffffff);
+ let f = |sh: u32| { let c = ((v >> sh) & 0xff) as f64; (c + (255.0 - c) * t) as u32 };
+ format!("#{:02x}{:02x}{:02x}", f(16), f(8), f(0))
+}
+
+// ── kill-on-exit job (whole tree dies with the orchestrator) ───────────────────
+static JOB: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
+fn job() -> HANDLE {
+ let raw = *JOB.get_or_init(|| unsafe {
+ let h = CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW");
+ let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
+ info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+ let _ = SetInformationJobObject(h, JobObjectExtendedLimitInformation,
+ &info as *const _ as *const core::ffi::c_void,
+ std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32);
+ h.0 as usize
+ });
+ HANDLE(raw as *mut core::ffi::c_void)
+}
+fn assign_to_job(child: &Child) {
+ use std::os::windows::io::AsRawHandle;
+ unsafe { let _ = AssignProcessToJobObject(job(), HANDLE(child.as_raw_handle() as *mut core::ffi::c_void)); }
+}
+
+fn main() {
+ let demo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf();
+ let repo_root = demo_root.parent().unwrap().parent().unwrap().to_path_buf();
+
+ let cua = std::env::var("CUA_DRIVER_EXE").map(PathBuf::from)
+ .unwrap_or_else(|_| repo_root.join("libs/cua-driver/rust/target/debug/cua-driver.exe"));
+ let app = std::env::var("JUKEBOX_APP_EXE").map(PathBuf::from)
+ .unwrap_or_else(|_| demo_root.join("target/debug/jukebox-app.exe"));
+ if !cua.exists() { eprintln!("cua-driver.exe not found at {cua:?}"); std::process::exit(1); }
+ if !app.exists() { eprintln!("jukebox-app.exe not found at {app:?} β€” run `cargo build` first"); std::process::exit(1); }
+
+ // Raise the system timer resolution to 1ms so the per-note `thread::sleep`
+ // that schedules each click is accurate to ~1ms instead of Windows' default
+ // ~15ms β€” the dominant residual timing jitter once the per-note process
+ // spawn was removed. Process-global; the OS restores it on exit.
+ unsafe { let _ = windows::Win32::Media::timeBeginPeriod(1); }
+
+ let args: Vec<String> = std::env::args().collect();
+ let auto = args.iter().any(|a| a == "--auto");
+ let midi_path = args.iter().skip(1).find(|a| a.to_lowercase().ends_with(".mid") || a.to_lowercase().ends_with(".midi"));
+ let glide_ms: u64 = std::env::var("JUKEBOX_GLIDE_MS").ok().and_then(|s| s.parse().ok()).unwrap_or(200);
+ // The dominant actuation latency IS the glide, so default the pre-roll to it
+ // (the per-voice EMA refines from there); overridable with JUKEBOX_LEAD_MS.
+ let lead = Duration::from_millis(std::env::var("JUKEBOX_LEAD_MS").ok().and_then(|s| s.parse().ok()).unwrap_or(glide_ms));
+
+ let mut song = match midi_path {
+ Some(p) => match load_midi(p) { Ok(s) => s, Err(e) => { eprintln!("[orch] MIDI parse failed ({e}); using demo song"); demo_song() } },
+ None => demo_song(),
+ };
+ assign_roles(&mut song);
+ if song.tracks.is_empty() { eprintln!("[orch] no tracks with notes"); std::process::exit(1); }
+ eprintln!("[orch] {} parts @ {} bpm, {:.1}s:", song.tracks.len(), song.bpm, song.dur_sec);
+ for t in &song.tracks {
+ let kind = match t.kind { Kind::Drums => "drums", Kind::Pad => "pad", Kind::Keys => "keys" };
+ let lo = t.notes.iter().map(|n| n.pitch as i32).min().unwrap_or(0);
+ let hi = t.notes.iter().map(|n| n.pitch as i32).max().unwrap_or(0);
+ let range = hi - lo + 1;
+ let fit = if t.kind == Kind::Keys && range > t.span {
+ format!(" RANGE {range}st > strip {} β†’ {} note(s) clamped", t.span,
+ t.notes.iter().filter(|n| (n.pitch as i32 - t.root) >= t.span).count())
+ } else { String::new() };
+ eprintln!(" Β· {:<22} {:<6} {:<8} {:>4} notes range {}-{} ({}st), strip {}st{}",
+ t.name, kind, t.wave, t.notes.len(), lo, hi, range, t.span, fit);
+ }
+
+ // Reap any leftover daemon from a previous run FIRST. Its job object only
+ // fires KILL_ON_JOB_CLOSE when the *owning* orchestrator exits cleanly, so a
+ // hard-killed run can leave a `cua-driver serve` alive β€” and a stale daemon
+ // keeps its own full-screen overlay, whose agent0..N cursors stack on top of
+ // this run's at the same (deterministic) coordinates. That's what makes it
+ // look like more than one cursor per window. Start from a clean slate.
+ for img in ["cua-driver.exe", "jukebox-app.exe"] {
+ let _ = Command::new("taskkill").args(["/F", "/IM", img])
+ .stdout(Stdio::null()).stderr(Stdio::null()).status();
+ }
+ // Give the killed daemon time to release its named pipe before we start a
+ // fresh one β€” too short a wait races the pipe handoff and can wedge the new
+ // daemon's first connections (observed as a one-off ~10s-late performance).
+ thread::sleep(Duration::from_millis(800));
+
+ // Start the cua-driver daemon.
+ eprintln!("[orch] starting cua-driver daemon…");
+ let mut daemon = Command::new(&cua).arg("serve").stdout(Stdio::null()).stderr(Stdio::null())
+ .spawn().expect("spawn cua-driver serve");
+ assign_to_job(&daemon);
+ thread::sleep(Duration::from_millis(1500));
+
+ // Launch the controller (foreground) + one instrument window per track.
+ let mut controller = Command::new(&app)
+ .args(["controller", "--title", "CUA JUKEBOX β€” Transport",
+ "--color", "#3bd0ff",
+ "--bpm", &song.bpm.to_string(),
+ "--dur-ms", &((song.dur_sec * 1000.0) as u64).to_string(),
+ "--tracks", &song.tracks.iter().map(|t| format!("{}|{}", t.name, t.color)).collect::<Vec<_>>().join(",")])
+ .stdout(Stdio::piped()).stderr(Stdio::null()).spawn().expect("spawn controller");
+ assign_to_job(&controller);
+ let controller_out = controller.stdout.take().unwrap();
+
+ let mut kids: Vec<Child> = Vec::new();
+ for t in &song.tracks {
+ let mut c = Command::new(&app);
+ c.args(["instrument",
+ "--title", &t.title,
+ "--kind", match t.kind { Kind::Keys => "keys", Kind::Drums => "drums", Kind::Pad => "pad" },
+ "--wave", t.wave,
+ "--color", t.color,
+ "--label", &t.name,
+ "--root", &t.root.to_string(),
+ "--span", &t.span.to_string()]);
+ if let Ok(ch) = c.stdout(Stdio::null()).stderr(Stdio::null()).spawn() { assign_to_job(&ch); kids.push(ch); }
+ }
+
+ thread::sleep(Duration::from_millis(1400)); // window warmup
+
+ // Discover HWNDs/pids.
+ for t in song.tracks.iter_mut() {
+ if let Some((h, pid)) = find_window_by_title(&t.title) { t.hwnd = h; t.pid = pid; }
+ else { eprintln!("[orch] (warn) no window for '{}'", t.title); }
+ }
+ song.tracks.retain(|t| !t.hwnd.0.is_null());
+ let controller_hwnd = find_window_by_title("CUA JUKEBOX β€” Transport").map(|(h, _)| h);
+
+ // Layout: a 600Γ—80 transport bar above a tight grid of fixed 200Γ—160 tiles,
+ // packed GAP px apart and centered in the work area.
+ const TILE_W: i32 = 200;
+ const TILE_H: i32 = 160;
+ const CTRL_W: i32 = 600;
+ const CTRL_H: i32 = 80;
+ const GAP: i32 = 6;
+ let wa = work_area();
+ let n = song.tracks.len() as i32;
+ let cols = (n as f64).sqrt().ceil().max(1.0) as i32;
+ let rows = (n + cols - 1) / cols.max(1);
+ let grid_w = cols * TILE_W + (cols - 1) * GAP;
+ let grid_h = rows * TILE_H + (rows - 1) * GAP;
+ let block_w = grid_w.max(CTRL_W);
+ let block_h = CTRL_H + GAP + grid_h;
+ let ox = wa.left + (((wa.right - wa.left) - block_w) / 2).max(0);
+ let oy = wa.top + (((wa.bottom - wa.top) - block_h) / 2).max(0);
+ if let Some(ch) = controller_hwnd {
+ place(ch, ox + (block_w - CTRL_W) / 2, oy, CTRL_W, CTRL_H, true);
+ }
+ let gy = oy + CTRL_H + GAP;
+ let gx = ox + (block_w - grid_w) / 2;
+ for (i, t) in song.tracks.iter().enumerate() {
+ let (col, row) = (i as i32 % cols, i as i32 / cols);
+ place(t.hwnd, gx + col * (TILE_W + GAP), gy + row * (TILE_H + GAP), TILE_W, TILE_H, false);
+ }
+ if let Some(ch) = controller_hwnd { unsafe { let _ = SetForegroundWindow(ch); } }
+ thread::sleep(Duration::from_millis(700));
+
+ // Build per-track cursor POOLS and arm each member. A track usually needs
+ // one cursor, but when notes land within the glide window (e.g. a chord) the
+ // first cursor is still mid-glide, so the next free pool cursor takes the
+ // note β€” growing the pool. Every member of a track is forced to the SAME
+ // colour (via set_agent_cursor_style's gradient β€” cursor_color on
+ // set_agent_cursor_motion is registry-only on the Windows overlay and would
+ // not repaint), so a chord fans out into several identically-coloured
+ // cursors on that one window.
+ // A cursor is "occupied" for the whole click β€” the glide PLUS the dispatch
+ // overhead (IPC + click-post + arrival frame), not just the glide. Sizing
+ // the pool against the real click duration means any track whose notes are
+ // closer together than that spawns enough cursors to keep up, instead of one
+ // cursor falling progressively behind.
+ let busy_sec = (glide_ms as f64 + 130.0) / 1000.0;
+ let mut voices: Vec<Voice> = Vec::new();
+ for t in &song.tracks {
+ let (assign, pool) = assign_pool(&t.notes, busy_sec, MAX_POOL);
+ let tip = lighten(t.color, 0.5);
+ for k in 0..pool {
+ let session = format!("{}_{}", t.session, k);
+ let _ = run_call(&cua, "set_agent_cursor_enabled",
+ &format!(r#"{{"enabled":true,"session":"{session}"}}"#));
+ // Fixed 100ms glide so every actuation lands on the beat regardless
+ // of travel distance (and so "occupied for the glide window" has a
+ // known length the pool sizes against).
+ // turn_radius 40 = half the default 80 β†’ tighter glide curves, which
+ // read better in the small tiles.
+ let _ = run_call(&cua, "set_agent_cursor_motion", &format!(
+ r#"{{"session":"{session}","cursor_label":"{}","cursor_size":15,"glide_duration_ms":{glide_ms},"spring":1.0,"arc_size":0.08,"dwell_after_click_ms":0,"idle_hide_ms":0,"turn_radius":40}}"#,
+ t.name));
+ let _ = run_call(&cua, "set_agent_cursor_style", &format!(
+ r#"{{"session":"{session}","gradient_colors":["{tip}","{}"],"bloom_color":"{}"}}"#,
+ t.color, t.color));
+ let notes_k: Vec<Note> = t.notes.iter().zip(assign.iter())
+ .filter(|(_, &a)| a == k).map(|(n, _)| n.clone()).collect();
+ voices.push(Voice {
+ pid: t.pid,
+ hwnd_addr: t.hwnd.0 as isize,
+ session,
+ kind: t.kind,
+ root: t.root,
+ span: t.span,
+ notes: Arc::new(notes_k),
+ });
+ }
+ if pool > 1 {
+ eprintln!("[orch] {:<10} pool={} same-colour cursors (notes closer than {:.0}ms)", t.name, pool, busy_sec * 1000.0);
+ }
+ }
+ let plans: Arc<Vec<Voice>> = Arc::new(voices);
+
+ let cua = Arc::new(cua);
+ let playing = Arc::new(AtomicBool::new(false));
+ let generation = Arc::new(AtomicU64::new(0));
+ let dur_sec = song.dur_sec;
+ let base_lead_ms = lead.as_secs_f64() * 1000.0;
+ // `timing` collects the signed per-note error (actualβˆ’scheduled) for the
+ // end-of-song report; cleared each performance. The adaptive lead itself is
+ // PER-VOICE (a thread-local EMA below), so one congested track β€” e.g. the
+ // Pad pool firing a triad β€” can't skew the lead of the tight single-cursor
+ // tracks.
+ let timing = Arc::new(std::sync::Mutex::new(Vec::<f64>::new()));
+ // Pause/resume: `offset` is the song-second to (re)start from; `seg` records
+ // the wall-clock start + offset of the running segment so PAUSE can compute
+ // where we are. STOP resets offset to 0; PAUSE saves the current position.
+ let offset = Arc::new(std::sync::Mutex::new(0.0_f64));
+ let seg = Arc::new(std::sync::Mutex::new((Instant::now(), 0.0_f64)));
+
+ let start_perf = {
+ let plans = plans.clone();
+ let cua = cua.clone();
+ let playing = playing.clone();
+ let generation = generation.clone();
+ let timing = timing.clone();
+ let seg = seg.clone();
+ move |off: f64| {
+ playing.store(true, Ordering::SeqCst);
+ let g = generation.fetch_add(1, Ordering::SeqCst) + 1;
+ let start = Instant::now();
+ *seg.lock().unwrap() = (start, off);
+ timing.lock().unwrap().clear();
+ eprintln!("[orch] β–Ά performing from {off:.1}s β€” {} cursors actuating in the background", plans.len());
+ for ti in 0..plans.len() {
+ let plans = plans.clone();
+ let cua = cua.clone();
+ let playing = playing.clone();
+ let generation = generation.clone();
+ let timing = timing.clone();
+ thread::spawn(move || {
+ let p = &plans[ti];
+ let hwnd = HWND(p.hwnd_addr as *mut core::ffi::c_void);
+ // Thread-local adaptive lead (ms) for THIS voice only,
+ // warm-started at the ~constant dispatch overhead on top of
+ // the glide (IPC + click-post + the arrival frame) so even
+ // the first notes land near the beat; the EMA refines it.
+ let mut corr = 110.0_f64;
+ // One persistent daemon connection for this whole track β€”
+ // pipelines every click with no per-note process spawn (that
+ // spawn was the entire timing-jitter source). The daemon runs
+ // a handler task per connection, so the 6 tracks actuate
+ // concurrently; within a track, clicks are lock-step (the
+ // response marks the actuation moment we measure against).
+ let pipe = std::env::var("CUA_DRIVER_PIPE")
+ .unwrap_or_else(|_| r"\\.\pipe\cua-driver".into());
+ let mut conn = DaemonConn::open(&pipe);
+ for (idx, note) in p.notes.iter().enumerate() {
+ if !playing.load(Ordering::SeqCst) || generation.load(Ordering::SeqCst) != g { return; }
+ if note.t < off { continue; } // already played before the resume point
+ // Fire early by the adaptive lead so the actuation lands
+ // on the note's scheduled time despite the glide latency.
+ // `start` represents song-time `off`, so schedule at the
+ // note's time minus that offset.
+ let eff_lead = (base_lead_ms + corr).max(0.0);
+ let due = start + Duration::from_secs_f64(note.t - off);
+ let fire = due.checked_sub(Duration::from_secs_f64(eff_lead / 1000.0)).unwrap_or(due);
+ let now = Instant::now();
+ if fire > now { thread::sleep(fire - now); }
+ if !playing.load(Ordering::SeqCst) || generation.load(Ordering::SeqCst) != g { return; }
+ let (xf, yf) = target(p, idx, note.pitch);
+ if let Some((x, y)) = client_rel_to_local_px(hwnd, xf, yf) {
+ let args = format!(
+ r#"{{"pid":{},"window_id":{},"x":{},"y":{},"session":"{}"}}"#,
+ p.pid, p.hwnd_addr, x, y, p.session);
+ let ok = match conn.as_mut() { Some(c) => c.call("click", &args), None => false };
+ if !ok {
+ // Reconnect once, else fall back to a one-shot call.
+ conn = DaemonConn::open(&pipe);
+ match conn.as_mut() {
+ Some(c) => { let _ = c.call("click", &args); }
+ None => { let _ = run_call(&cua, "click", &args); }
+ }
+ }
+ // Response received β‡’ the click landed (actuation moment).
+ let err_ms = (start.elapsed().as_secs_f64() - note.t) * 1000.0;
+ timing.lock().unwrap().push(err_ms);
+ // EMA drives the mean error to zero: eff_lead = base +
+ // corr and err = latency βˆ’ eff_lead, so corr += Ξ±Β·err
+ // converges corr β†’ latency βˆ’ base. Clamp tight so a
+ // transient stall can't integrate the lead away. Ξ± is
+ // fairly high so short voices (a few-note pool member)
+ // converge within the song.
+ corr = (corr + 0.30 * err_ms).clamp(-base_lead_ms, 600.0);
+ }
+ }
+ });
+ }
+ // End-of-song timing report (one per performance/generation).
+ let timing = timing.clone();
+ let generation = generation.clone();
+ thread::spawn(move || {
+ thread::sleep(Duration::from_secs_f64((dur_sec - off).max(0.0) + 0.8));
+ if generation.load(Ordering::SeqCst) != g { return; }
+ report_timing(&timing.lock().unwrap(), base_lead_ms);
+ });
+ }
+ };
+
+ if auto {
+ thread::sleep(Duration::from_millis(800));
+ start_perf(0.0);
+ } else {
+ eprintln!("[orch] ready β€” click β–Ά PLAY in the Transport window to start the performance.");
+ }
+
+ // React to the transport's PLAY / PAUSE / STOP (the human's one action that
+ // drives the whole coordinated fleet β€” like the master window in the sibling
+ // demo). PLAY resumes from the held offset; PAUSE freezes the position; STOP
+ // resets to the top.
+ use std::io::{BufRead, BufReader};
+ let reader = BufReader::new(controller_out);
+ for line in reader.lines().map_while(Result::ok) {
+ // Drag-drop a .mid onto the transport β†’ restart the whole demo with that
+ // track. Re-exec a fresh orchestrator (detached, outside our job object,
+ // so it survives our exit); it reaps our daemon/windows on startup and
+ // comes up READY on the dropped file (no --auto β€” the user presses β–Ά).
+ if let Some(path) = line.trim_end().strip_prefix("LOAD\t") {
+ eprintln!("[orch] ↻ restarting with {path}");
+ if let Ok(exe) = std::env::current_exe() {
+ let _ = Command::new(exe).arg(path)
+ .stdout(Stdio::null()).stderr(Stdio::null()).spawn();
+ }
+ break; // fall through to teardown; the new orchestrator takes over
+ }
+ match line.trim() {
+ // Idempotent: ignore PLAY while already performing (resume still works
+ // β€” PAUSE clears `playing` first), so a stray PLAY can't double-start.
+ "PLAY" if !playing.load(Ordering::SeqCst) => { let off = *offset.lock().unwrap(); start_perf(off); }
+ "PAUSE" => {
+ playing.store(false, Ordering::SeqCst);
+ let (rs, off) = *seg.lock().unwrap();
+ let pos = off + rs.elapsed().as_secs_f64();
+ *offset.lock().unwrap() = pos;
+ eprintln!("[orch] ⏸ paused at {pos:.1}s");
+ }
+ "STOP" => {
+ playing.store(false, Ordering::SeqCst);
+ *offset.lock().unwrap() = 0.0;
+ eprintln!("[orch] β–  stopped");
+ }
+ _ => {}
+ }
+ }
+
+ // Controller closed β†’ tear everything down (job object kills the tree too).
+ playing.store(false, Ordering::SeqCst);
+ let _ = controller.kill();
+ for mut k in kids { let _ = k.kill(); }
+ let _ = daemon.kill();
+}
+
+// ── song construction ──────────────────────────────────────────────────────────
+
+/// True if a track name reads like a full drum kit (vs a single drum part like
+/// "Kick", which stays a single pad).
+fn name_is_kit(name: &str) -> bool {
+ let n = name.to_lowercase();
+ ["drum kit", "drumkit", "drums", "percussion", "drum set"].iter().any(|s| n.contains(s))
+}
+
+/// Map a MIDI channel-10 drum note to a 3-zone pad: 0=kick, 1=snare/clap/tom,
+/// 2=hat/cymbal/perc. (General MIDI percussion key map.)
+fn drum_zone(pitch: u8) -> i32 {
+ match pitch {
+ 35 | 36 => 0, // bass/kick drums
+ 37 | 38 | 39 | 40 | 41 | 43 | 45 | 47 | 48 | 50 => 1, // snare, clap, rim, toms
+ _ => 2, // hats, cymbals, perc
+ }
+}
+
+/// Infer a melodic track's minigame + synth voice from its (lower-cased) name.
+/// Wave keywords (square/saw/triangle/8-bit/chip/synth/scifi/…) are matched
+/// explicitly so chiptune/GM names map to a fitting voice instead of plain sine.
+fn infer(name: &str) -> (Kind, &'static str) {
+ let n = name.to_lowercase();
+ let has = |k: &[&str]| k.iter().any(|s| n.contains(s));
+ // Single drum parts (not full kits) β†’ one-shot pad.
+ if has(&["kick", "bass drum", "bd"]) { (Kind::Pad, "kick") }
+ else if has(&["snare", "clap", "sd"]) { (Kind::Pad, "snare") }
+ else if has(&["hi-hat", "hihat", "hat", "cymbal", "ride", "shaker", "tom "]) { (Kind::Pad, "hat") }
+ // Bass before the wave words so "Bass Guitar" is a bass, not a saw lead.
+ else if has(&["bass"]) { (Kind::Keys, "saw") }
+ // Explicit waveform names FIRST (so "8-Bit Triangle" is a triangle, not
+ // caught by the chip→square fallback below).
+ else if has(&["sawtooth", "saw"]) { (Kind::Keys, "saw") }
+ else if has(&["triangle"]) { (Kind::Keys, "triangle") }
+ else if has(&["square", "pulse"]) { (Kind::Keys, "square") }
+ // Generic chiptune β†’ square.
+ else if has(&["8-bit", "8bit", "chip", "nes"]) { (Kind::Keys, "square") }
+ // Timbre families.
+ else if has(&["lead", "synth", "scifi", "sci-fi", "guitar", "trumpet", "sax", "brass", "pluck"]) { (Kind::Keys, "square") }
+ else if has(&["pad", "string", "choir", "organ", "ambient", "smooth", "warm"]) { (Kind::Keys, "triangle") }
+ else if has(&["arp", "bell", "key", "piano", "mallet", "celesta", "harp"]) { (Kind::Keys, "triangle") }
+ else { (Kind::Keys, "sine") }
+}
+
+fn assign_roles(song: &mut Song) {
+ song.tracks.truncate(MAX_TRACKS);
+ for (i, t) in song.tracks.iter_mut().enumerate() {
+ let (kind, wave) = if t.is_drum { (Kind::Drums, "kick") } else { infer(&t.name) };
+ t.kind = kind;
+ t.wave = wave;
+ let (sess, hex) = SESSIONS[i % SESSIONS.len()];
+ t.color = hex;
+ t.session = sess.to_string();
+ t.title = format!("JUKEBOX {i:02} β€” {}", t.name);
+ // Fit the pitch strip to the track's actual range so wide melodies are
+ // played accurately instead of being clamped to the top key: root = the
+ // lowest note, span = the full range (min 12 semitones for a usable
+ // strip, capped at 48 = 4 octaves so the keys don't get microscopic).
+ let lo = t.notes.iter().map(|n| n.pitch as i32).min().unwrap_or(48);
+ let hi = t.notes.iter().map(|n| n.pitch as i32).max().unwrap_or(72);
+ t.root = lo;
+ t.span = (hi - lo + 1).clamp(12, 48);
+ }
+}
+
+fn mk_track(name: &str, notes: Vec<Note>) -> Track {
+ Track { name: name.into(), notes, kind: Kind::Keys, wave: "sine", root: 48, span: KEYS_SPAN,
+ color: "#888", session: String::new(), title: String::new(), hwnd: HWND::default(), pid: 0,
+ is_drum: false }
+}
+
+/// Generated 8-bar, 6-part loop at 120 bpm β€” a 1:1 feel-port of the HTML demo
+/// song so the grid moves immediately even with no .mid file.
+fn demo_song() -> Song {
+ let q = 0.5_f64; // quarter-note seconds at 120 bpm
+ let bars = 8;
+ let roots = [36, 36, 43, 41];
+ let sc = [0, 2, 4, 7, 9, 12];
+ let (mut bass, mut kick, mut hat, mut pad, mut arp, mut lead) =
+ (vec![], vec![], vec![], vec![], vec![], vec![]);
+ for b in 0..bars {
+ let o = b as f64 * q * 4.0;
+ let r = roots[b % 4] as u8;
+ bass.push(Note { t: o, pitch: r, vel: 100 });
+ bass.push(Note { t: o + q, pitch: r, vel: 80 });
+ bass.push(Note { t: o + 2.0 * q, pitch: r + 7, vel: 95 });
+ bass.push(Note { t: o + 3.0 * q, pitch: r, vel: 80 });
+ for k in 0..4 { kick.push(Note { t: o + k as f64 * q, pitch: 36, vel: 110 }); }
+ for h in 0..8 { hat.push(Note { t: o + h as f64 * q / 2.0, pitch: 42, vel: if h % 2 == 0 { 80 } else { 55 } }); }
+ // Triads struck simultaneously β€” three notes at the same instant means
+ // one cursor can't cover them within the 100ms glide, so the Pad track
+ // grows a 3-cursor same-colour pool that fans out across the strip.
+ for iv in [12, 16, 19] { pad.push(Note { t: o, pitch: r + iv, vel: 52 }); }
+ for iv in [12, 15, 19] { pad.push(Note { t: o + 2.0 * q, pitch: r + iv, vel: 50 }); }
+ for a in 0..8 { arp.push(Note { t: o + a as f64 * q / 2.0, pitch: r + 24 + sc[(a + b) % 6] as u8, vel: 70 }); }
+ if b % 2 == 1 { for l in 0..4 { lead.push(Note { t: o + l as f64 * q, pitch: r + 24 + sc[(l * 2) % 6] as u8, vel: 88 }); } }
+ }
+ Song {
+ tracks: vec![
+ mk_track("Bass", bass), mk_track("Kick", kick), mk_track("Hat", hat),
+ mk_track("Pad", pad), mk_track("Arp", arp), mk_track("Lead", lead),
+ ],
+ bpm: 120,
+ dur_sec: bars as f64 * q * 4.0,
+ }
+}
+
+/// Parse a Standard MIDI File into per-track note lists. Single tempo (first
+/// tempo event wins) β€” good enough for the demo; documented in the README.
+fn load_midi(path: &str) -> Result<Song, String> {
+ use midly::{MetaMessage, MidiMessage, Smf, Timing, TrackEventKind};
+ let data = std::fs::read(path).map_err(|e| e.to_string())?;
+ let smf = Smf::parse(&data).map_err(|e| e.to_string())?;
+ let ppq = match smf.header.timing { Timing::Metrical(t) => t.as_int() as f64, _ => 480.0 };
+ // First tempo found across all tracks (us per quarter-note).
+ let mut tempo = 500_000.0_f64;
+ 'outer: for tr in &smf.tracks {
+ for ev in tr {
+ if let TrackEventKind::Meta(MetaMessage::Tempo(t)) = ev.kind { tempo = t.as_int() as f64; break 'outer; }
+ }
+ }
+ let spt = (tempo / 1e6) / ppq; // seconds per tick
+ let mut tracks = Vec::new();
+ let mut dur = 0.0_f64;
+ for tr in &smf.tracks {
+ let mut tick = 0u64;
+ let mut name = String::new();
+ // Key by (channel, pitch) so overlapping same-pitch notes on different
+ // channels don't collide, and polyphonic same-pitch (rare) is handled.
+ let mut on: std::collections::HashMap<(u8, u8), (f64, u8)> = std::collections::HashMap::new();
+ let mut notes = Vec::new();
+ let mut drum_notes = 0usize; // notes seen on MIDI channel 10 (index 9)
+ for ev in tr {
+ tick += ev.delta.as_int() as u64;
+ let now = tick as f64 * spt;
+ match ev.kind {
+ TrackEventKind::Meta(MetaMessage::TrackName(bytes)) =>
+ if name.is_empty() { name = String::from_utf8_lossy(bytes).trim().to_string(); },
+ TrackEventKind::Midi { channel, message: MidiMessage::NoteOn { key, vel } } => {
+ let ch = channel.as_int();
+ let pitch = key.as_int();
+ if vel.as_int() > 0 {
+ on.insert((ch, pitch), (now, vel.as_int()));
+ if ch == 9 { drum_notes += 1; }
+ }
+ else if let Some((t0, v)) = on.remove(&(ch, pitch)) { notes.push(Note { t: t0, pitch, vel: v }); dur = dur.max(now); }
+ }
+ TrackEventKind::Midi { channel, message: MidiMessage::NoteOff { key, .. } } => {
+ if let Some((t0, v)) = on.remove(&(channel.as_int(), key.as_int())) { notes.push(Note { t: t0, pitch: key.as_int(), vel: v }); dur = dur.max(now); }
+ }
+ _ => {}
+ }
+ }
+ if !notes.is_empty() {
+ notes.sort_by(|a, b| a.t.partial_cmp(&b.t).unwrap());
+ let idx = tracks.len();
+ let nm = if name.is_empty() { format!("Track {}", idx + 1) } else { name.clone() };
+ // Percussion if most notes were on channel 10, or the name is a kit.
+ let is_drum = drum_notes * 2 > notes.len() || name_is_kit(&nm);
+ let mut t = mk_track(&nm, notes);
+ t.is_drum = is_drum;
+ tracks.push(t);
+ }
+ }
+ if tracks.is_empty() { return Err("no note tracks".into()); }
+ let bpm = (60.0 / (tempo / 1e6)).round() as u32;
+ Ok(Song { tracks, bpm, dur_sec: dur + 0.5 })
+}
+
+// ── cua-driver call + Win32 helpers ─────────────────────────────────────────────
+
+/// Print the per-note timing diff: how far each actuation landed from its
+/// scheduled beat (actual βˆ’ scheduled, ms). `mean`β†’0 means the adaptive lead
+/// has cancelled the systematic latency; `sd`/`p90` are the residual jitter
+/// (dominated by per-note `cua-driver call` subprocess spawn).
+fn report_timing(errs: &[f64], base_lead_ms: f64) {
+ if errs.is_empty() { eprintln!("[timing] no notes measured"); return; }
+ let n = errs.len();
+ let mut s = errs.to_vec();
+ s.sort_by(|a, b| a.partial_cmp(b).unwrap());
+ let mean = errs.iter().sum::<f64>() / n as f64;
+ let sd = (errs.iter().map(|e| (e - mean).powi(2)).sum::<f64>() / n as f64).sqrt();
+ let absmean = errs.iter().map(|e| e.abs()).sum::<f64>() / n as f64;
+ let pct = |q: f64| s[((n as f64 * q) as usize).min(n - 1)];
+ eprintln!(
+ "[timing] n={n} mean={mean:+.1}ms |mean|={absmean:.1}ms sd={sd:.1}ms \
+ median={:+.1}ms p10={:+.1} p90={:+.1} min={:+.1} max={:+.1}",
+ pct(0.5), pct(0.1), pct(0.9), s[0], s[n - 1],
+ );
+ eprintln!(
+ "[timing] base lead {:.0}ms, refined per-voice; + β‡’ late, βˆ’ β‡’ early.",
+ base_lead_ms,
+ );
+}
+
+fn run_call(cua: &PathBuf, tool: &str, json: &str) -> bool {
+ Command::new(cua).arg("call").arg(tool).arg(json)
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .status().map(|s| s.success()).unwrap_or(false)
+}
+
+/// A persistent connection to the `cua-driver serve` daemon over its
+/// line-delimited-JSON named pipe (`\\.\pipe\cua-driver`). The daemon handles
+/// many requests per connection, so holding ONE open per track lets us pipeline
+/// every note's click without paying a `cua-driver call` process spawn per note
+/// β€” that spawn (tens of ms, high variance) was the entire timing-jitter
+/// source. One connection per track also means tracks dispatch concurrently
+/// server-side (a handler task per connection).
+struct DaemonConn {
+ writer: std::fs::File,
+ reader: std::io::BufReader<std::fs::File>,
+}
+
+impl DaemonConn {
+ fn open(pipe: &str) -> Option<DaemonConn> {
+ use std::time::Instant as I;
+ let deadline = I::now() + Duration::from_secs(3);
+ loop {
+ match std::fs::OpenOptions::new().read(true).write(true).open(pipe) {
+ Ok(f) => {
+ let reader = std::io::BufReader::new(f.try_clone().ok()?);
+ return Some(DaemonConn { writer: f, reader });
+ }
+ Err(_) if I::now() < deadline => thread::sleep(Duration::from_millis(40)),
+ Err(_) => return None,
+ }
+ }
+ }
+
+ /// Send one `call` and block for its response line. `args` is a JSON object
+ /// literal. Returns whether the tool reported ok. The daemon processes one
+ /// request per connection at a time and replies in order, so this lockstep
+ /// is safe and the response marks the actuation moment.
+ fn call(&mut self, tool: &str, args: &str) -> bool {
+ use std::io::{BufRead, Write};
+ let line = format!(r#"{{"method":"call","name":"{tool}","args":{args}}}"#) + "\n";
+ if self.writer.write_all(line.as_bytes()).is_err() { return false; }
+ if self.writer.flush().is_err() { return false; }
+ let mut resp = String::new();
+ match self.reader.read_line(&mut resp) {
+ Ok(0) | Err(_) => false,
+ Ok(_) => resp.contains("\"ok\":true"),
+ }
+ }
+}
+
+/// Window-local (x,y) fraction the cursor should click for a note. Melodic
+/// strips map pitch β†’ key cell (fixed y). Drum pads have no pitch, so we hop the
+/// click around the pad each hit (deterministic per note index) β€” otherwise the
+/// cursor would click dead-centre every time and look static; the hop makes the
+/// 100ms glide visible. Must match jukebox-app's WX0/WX1/WY0/WY1/KEYS_SPAN.
+fn target(p: &Voice, idx: usize, pitch: u8) -> (f64, f64) {
+ match p.kind {
+ Kind::Keys => {
+ let span = p.span.max(1);
+ let semi = (pitch as i32 - p.root).clamp(0, span - 1);
+ (WX0 + (semi as f64 + 0.5) / span as f64 * (WX1 - WX0), (WY0 + WY1) / 2.0)
+ }
+ Kind::Drums => {
+ // Click the kick / snare / hat zone this drum note belongs to.
+ let z = drum_zone(pitch);
+ (WX0 + (z as f64 + 0.5) / 3.0 * (WX1 - WX0), (WY0 + WY1) / 2.0)
+ }
+ Kind::Pad => {
+ let h = (idx as u32).wrapping_mul(2_654_435_761);
+ let xf = 0.30 + (h % 1000) as f64 / 1000.0 * 0.40;
+ let yf = WY0 + 0.10 + ((h / 1000) % 1000) as f64 / 1000.0 * (WY1 - WY0 - 0.20);
+ (xf, yf)
+ }
+ }
+}
+
+fn work_area() -> RECT {
+ use windows::Win32::UI::WindowsAndMessaging::{SystemParametersInfoW, SPI_GETWORKAREA, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS};
+ let mut wa = RECT::default();
+ unsafe { let _ = SystemParametersInfoW(SPI_GETWORKAREA, 0, Some(&mut wa as *mut _ as *mut core::ffi::c_void), SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0)); }
+ if wa.right <= wa.left || wa.bottom <= wa.top {
+ use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};
+ unsafe { wa = RECT { left: 0, top: 0, right: GetSystemMetrics(SM_CXSCREEN), bottom: GetSystemMetrics(SM_CYSCREEN) }; }
+ }
+ wa
+}
+
+fn place(hwnd: HWND, x: i32, y: i32, w: i32, h: i32, activate: bool) {
+ if hwnd.0.is_null() { return; }
+ let flags = if activate { SWP_SHOWWINDOW } else { SWP_NOACTIVATE | SWP_SHOWWINDOW | SWP_NOZORDER };
+ unsafe { let _ = SetWindowPos(hwnd, HWND_TOP, x, y, w, h, flags); }
+}
+
+/// Client-relative (0..1) β†’ the click tool's window-local screenshot-pixel space
+/// (ClientToScreen minus the DWM extended-frame top-left + 1px inset). Same math
+/// as the multi-cursor demo.
+fn client_rel_to_local_px(hwnd: HWND, rx: f64, ry: f64) -> Option<(i32, i32)> {
+ unsafe {
+ let mut cr = RECT::default();
+ GetClientRect(hwnd, &mut cr).ok()?;
+ let mut pt = POINT { x: (rx * (cr.right - cr.left) as f64) as i32, y: (ry * (cr.bottom - cr.top) as f64) as i32 };
+ let _ = ClientToScreen(hwnd, &mut pt);
+ let mut dwm = RECT::default();
+ if DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &mut dwm as *mut _ as *mut core::ffi::c_void, std::mem::size_of::<RECT>() as u32).is_err() {
+ return Some((pt.x, pt.y));
+ }
+ Some((pt.x - dwm.left - 1, pt.y - dwm.top - 1))
+ }
+}
+
+// ── window discovery by title substring (case-insensitive) ──────────────────────
+struct Finder { needle: String, hwnd: HWND, pid: u32 }
+unsafe extern "system" fn enum_cb(hwnd: HWND, lparam: LPARAM) -> BOOL {
+ let f = &mut *(lparam.0 as *mut Finder);
+ if !IsWindowVisible(hwnd).as_bool() { return TRUE; }
+ let mut buf = [0u16; 256];
+ let n = GetWindowTextW(hwnd, &mut buf);
+ if n > 0 {
+ let title = String::from_utf16_lossy(&buf[..n as usize]);
+ if title.to_lowercase().contains(&f.needle.to_lowercase()) {
+ let mut pid = 0u32;
+ GetWindowThreadProcessId(hwnd, Some(&mut pid));
+ f.hwnd = hwnd; f.pid = pid;
+ return BOOL(0);
+ }
+ }
+ TRUE
+}
+fn find_window_by_title(needle: &str) -> Option<(HWND, u32)> {
+ let deadline = Instant::now() + Duration::from_secs(8);
+ loop {
+ let mut f = Finder { needle: needle.to_string(), hwnd: HWND::default(), pid: 0 };
+ unsafe { let _ = EnumWindows(Some(enum_cb), LPARAM(&mut f as *mut _ as isize)); }
+ if !f.hwnd.0.is_null() { return Some((f.hwnd, f.pid)); }
+ if Instant::now() > deadline { return None; }
+ thread::sleep(Duration::from_millis(300));
+ }
+}
diff --git a/demo/multi-cursor/.gitignore b/demo/multi-cursor/.gitignore
new file mode 100644
index 0000000000..bbbbe20a0e
--- /dev/null
+++ b/demo/multi-cursor/.gitignore
@@ -0,0 +1,8 @@
+# Rust
+/target/
+# .NET
+**/bin/
+**/obj/
+# Electron / Node
+electron/node_modules/
+electron/package-lock.json
diff --git a/demo/multi-cursor/Cargo.lock b/demo/multi-cursor/Cargo.lock
new file mode 100644
index 0000000000..8df441af11
--- /dev/null
+++ b/demo/multi-cursor/Cargo.lock
@@ -0,0 +1,312 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "adler2"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+
+[[package]]
+name = "autocfg"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+
+[[package]]
+name = "bitflags"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
+
+[[package]]
+name = "bytemuck"
+version = "1.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
+
+[[package]]
+name = "byteorder-lite"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
+
+[[package]]
+name = "crc32fast"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "fdeflate"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
+dependencies = [
+ "simd-adler32",
+]
+
+[[package]]
+name = "flate2"
+version = "1.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+dependencies = [
+ "crc32fast",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "image"
+version = "0.25.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
+dependencies = [
+ "bytemuck",
+ "byteorder-lite",
+ "moxcms",
+ "num-traits",
+ "png",
+]
+
+[[package]]
+name = "legacy-app"
+version = "0.1.0"
+dependencies = [
+ "windows",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
+[[package]]
+name = "moxcms"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
+dependencies = [
+ "num-traits",
+ "pxfm",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "orchestrator"
+version = "0.1.0"
+dependencies = [
+ "image",
+ "windows",
+]
+
+[[package]]
+name = "png"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"
+dependencies = [
+ "bitflags",
+ "crc32fast",
+ "fdeflate",
+ "flate2",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pxfm"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f"
+
+[[package]]
+name = "quote"
+version = "1.0.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "simd-adler32"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
+
+[[package]]
+name = "syn"
+version = "2.0.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "windows"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"
+dependencies = [
+ "windows-core",
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-result",
+ "windows-strings",
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.58.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-result"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
+dependencies = [
+ "windows-result",
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
diff --git a/demo/multi-cursor/Cargo.toml b/demo/multi-cursor/Cargo.toml
new file mode 100644
index 0000000000..bf84427611
--- /dev/null
+++ b/demo/multi-cursor/Cargo.toml
@@ -0,0 +1,6 @@
+[workspace]
+resolver = "2"
+members = ["legacy-app", "orchestrator"]
+
+[profile.release]
+opt-level = 2
diff --git a/demo/multi-cursor/README.md b/demo/multi-cursor/README.md
new file mode 100644
index 0000000000..496fee1387
--- /dev/null
+++ b/demo/multi-cursor/README.md
@@ -0,0 +1,89 @@
+# Multi-cursor background computer-use demo β€” "National Records System"
+
+A fleet of deliberately **legacy-looking government records terminals** (navy
+banner, `UNCLASSIFIED // FOR OFFICIAL USE ONLY` strip, function-key bar,
+green-screen records grid, status line) β€” the kind of internal agency app that,
+in the age of AI, has *no* automation integration. cua-driver automates them
+anyway.
+
+One human action in the foreground "master" terminal is replayed onto **four
+background terminals at the same time**, each driven by its own cua-driver
+session = its own uniquely-coloured agent cursor β€” with **no window ever
+raised** and **the user's mouse never moved**.
+
+It also proves cua-driver works **with or without an accessibility tree**: the
+five windows span five UI frameworks, and cua-driver's default dispatch
+auto-selects UIA-Invoke where an a11y tree exists and falls back to
+pixel/pointer-injection where it doesn't.
+
+## Layout (each window = Β½ work-width Γ— Β½ work-height)
+
+The four corners tile the taskbar-safe work area into quadrants; the master is
+centered, **overlapping all four**:
+
+```
+ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”
+ β”‚ Win32 GDI (NO a11y) β”‚ WinForms (.NET) β”‚
+ β”‚ crimson ● β”‚ amber ● β”‚
+ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚
+ β”‚ β”‚ MASTER β€” Win32 controls β”‚ ← you β”‚
+ β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”‚ (foreground, overlaps) │──────────────
+ β”‚ WPF (XAML)β”‚ β”‚ Electron β”‚
+ β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ mint_lime ● β”‚
+ β”‚ aqua ● β”‚ (Chromium) β”‚
+ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜
+```
+
+Click **SUBMIT** (or type a subject name then submit) in the center master:
+four coloured cursors glide onto the four corner terminals and commit the same
+record there β€” concurrently, in the background. Watch each corner's
+green-screen records grid grow and its `RECORDS:` counter tick up, without any
+corner ever coming to the front.
+
+## Frameworks (and what they exercise)
+
+| Window | Framework | Accessibility | cua-driver path |
+|---|---|---|---|
+| TL | Win32 + GDI (custom-drawn) | **none** | pixel hit-test β†’ PostMessage / pointer injection |
+| TR | .NET WinForms | MSAA/UIA | UIA Invoke |
+| BL | .NET WPF | UIA (XAML) | UIA Invoke (no foreground steal via WS_EX_NOACTIVATE) |
+| BR | Electron | UIA (Chromium) | UIA Invoke |
+| Center | Win32 standard controls | MSAA | (foreground; the human drives it) |
+
+## Build
+
+```powershell
+# from this directory
+cargo build # legacy-app + orchestrator (Rust)
+dotnet build dotnet/winforms/winforms.csproj # WinForms
+dotnet build dotnet/wpf/wpf.csproj # WPF
+npm install --prefix electron # Electron (downloads electron once)
+```
+Also build the driver once (repo root workspace):
+```powershell
+cargo build -p cua-driver --manifest-path ..\..\libs\cua-driver\rust\Cargo.toml
+```
+
+## Run
+
+```powershell
+.\target\debug\orchestrator.exe # human-driven: click/type in the center
+.\target\debug\orchestrator.exe --auto # self-playing: drives a TYPE+CLICK every few seconds
+```
+
+The orchestrator starts the cua-driver daemon, launches + positions all five
+windows, and fans every center action out to the four corners over four
+concurrent `cua-driver call` sessions (`crimson` / `amber` / `aqua` /
+`mint_lime` β†’ four cursor colours). Close the center window (or kill the
+orchestrator) to tear everything down β€” a Windows Job Object kills the whole
+tree, so nothing is orphaned.
+
+### Env overrides
+`CUA_DRIVER_EXE`, `LEGACY_APP_EXE`, `WINFORMS_EXE`, `WPF_EXE`, `ELECTRON_DIR`.
+
+## How the coloured cursors work
+cua-driver assigns each session a cursor colour by name (palette-name sessions
+like `crimson` pick that colour directly). Passing `"session":"<color>"` on each
+`click`/`type_text` call routes it to that session's overlay cursor, which
+glides to the target. Four sessions β†’ four cursors animating at once. See
+`docs/windows-background-input-re-plan.md` for the no-z-raise mechanism.
diff --git a/demo/multi-cursor/dotnet/winforms/Program.cs b/demo/multi-cursor/dotnet/winforms/Program.cs
new file mode 100644
index 0000000000..88759c4c59
--- /dev/null
+++ b/demo/multi-cursor/dotnet/winforms/Program.cs
@@ -0,0 +1,99 @@
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Windows.Forms;
+
+// "Meridian CRM β€” Account Record" : WinForms node (classic Win32 controls,
+// MSAA/UIA). Explicit fractional layout (dense, no gaps beyond borders).
+// cua-driver drives the Account Name field + the "Add Record" button.
+static class Program
+{
+ [STAThread]
+ static void Main()
+ {
+ Application.EnableVisualStyles();
+ Application.SetCompatibleTextRenderingDefault(false);
+
+ var f = new Form { Text = "Meridian CRM β€” Account Record [WinForms]", ClientSize = new Size(960, 600),
+ StartPosition = FormStartPosition.Manual, Font = new Font("Segoe UI", 9f), BackColor = Color.FromArgb(240,240,240) };
+
+ Label band(string t, Color bg, ContentAlignment a = ContentAlignment.MiddleLeft, FontStyle fs = FontStyle.Regular)
+ => new Label { Text = t, BackColor = bg, TextAlign = a, Font = new Font("Segoe UI", 9f, fs) };
+
+ var menu = band(" File Edit View Record Tools Help", Color.FromArgb(247,247,247));
+ var tool = band(" New Open Save Delete β”‚ β—€ Prev Next β–Ά β”‚ Find", Color.FromArgb(235,235,235));
+ var nameLbl = band(" Account Name", Color.FromArgb(240,240,240));
+ var typeLbl = band(" Account Type", Color.FromArgb(240,240,240));
+ var regionLbl = band(" Region", Color.FromArgb(240,240,240));
+ var prioLbl = band(" Priority (1-5)", Color.FromArgb(240,240,240));
+ var creditLbl = band(" Credit Limit", Color.FromArgb(240,240,240));
+ var nameBox = new TextBox { BorderStyle = BorderStyle.FixedSingle };
+ var typeCmb = new ComboBox { DropDownStyle = ComboBoxStyle.DropDownList, FlatStyle = FlatStyle.Flat };
+ typeCmb.Items.AddRange(new object[] { "Enterprise", "SMB", "Government", "Reseller" }); typeCmb.SelectedIndex = 0;
+ var regionCmb = new ComboBox { DropDownStyle = ComboBoxStyle.DropDownList, FlatStyle = FlatStyle.Flat };
+ regionCmb.Items.AddRange(new object[] { "North", "South", "EMEA", "APAC", "LATAM" }); regionCmb.SelectedIndex = 0;
+ var prio = new TrackBar { Minimum = 1, Maximum = 5, Value = 3, TickStyle = TickStyle.BottomRight };
+ var credit = new TrackBar { Minimum = 0, Maximum = 100, Value = 40, TickFrequency = 10, TickStyle = TickStyle.BottomRight };
+ var saveBtn = new Button { Text = "Add Record", FlatStyle = FlatStyle.System, Font = new Font("Segoe UI", 10f, FontStyle.Bold) };
+ var sigHdr = band(" Signature / Notes", Color.FromArgb(225,225,225), ContentAlignment.MiddleLeft, FontStyle.Bold);
+ var sig = new Panel { BackColor = Color.White, BorderStyle = BorderStyle.FixedSingle };
+ var grid = new DataGridView { AllowUserToAddRows = false, ReadOnly = true, BackgroundColor = Color.White,
+ BorderStyle = BorderStyle.FixedSingle, RowHeadersVisible = false, AllowUserToResizeRows = false,
+ ColumnHeadersHeightSizeMode = DataGridViewColumnHeadersHeightSizeMode.DisableResizing };
+ grid.Columns.Add("acct", "Account"); grid.Columns.Add("type", "Type"); grid.Columns.Add("region", "Region");
+ grid.Columns.Add("pri", "Pri"); grid.Columns.Add("credit", "Credit");
+ grid.Columns["acct"].AutoSizeMode = DataGridViewAutoSizeColumnMode.Fill;
+ var status = band(" Ready", Color.FromArgb(232,232,232));
+ var recLbl = band("Records: 0 USER: SYSTEM β–Œ CONNECTED ", Color.FromArgb(232,232,232), ContentAlignment.MiddleRight);
+
+ // line tool: one straight segment per press-drag-release (down→up)
+ var segs = new List<(Point a, Point b)>(); Point? down = null; Point cur = Point.Empty;
+ sig.MouseDown += (s, e) => { down = e.Location; cur = e.Location; };
+ sig.MouseMove += (s, e) => { if (down != null) { cur = e.Location; sig.Invalidate(); } };
+ sig.MouseUp += (s, e) => { if (down != null) { segs.Add((down.Value, e.Location)); down = null; sig.Invalidate(); } };
+ sig.Paint += (s, e) => { e.Graphics.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.AntiAlias;
+ using var p = new Pen(Color.MidnightBlue, 2); foreach (var sg in segs) e.Graphics.DrawLine(p, sg.a, sg.b);
+ if (down != null) { using var pp = new Pen(Color.FromArgb(110, 25, 25, 112), 1); e.Graphics.DrawLine(pp, down.Value, cur); } };
+
+ int records = 0;
+ saveBtn.Click += (s, e) =>
+ {
+ string nm = nameBox.Text.Trim(); if (nm.Length == 0) nm = "(unnamed)";
+ grid.Rows.Add(nm, typeCmb.Text, regionCmb.Text, prio.Value, "$" + (credit.Value * 1000));
+ if (grid.Rows.Count > 0) grid.FirstDisplayedScrollingRowIndex = grid.Rows.Count - 1;
+ records++; status.Text = $" Saved account '{nm}'."; recLbl.Text = $"Records: {records} USER: SYSTEM β–Œ CONNECTED ";
+ nameBox.Clear();
+ };
+
+ foreach (Control c in new Control[] { menu, tool, nameLbl, typeLbl, regionLbl, prioLbl, creditLbl,
+ nameBox, typeCmb, regionCmb, prio, credit, saveBtn, sigHdr, sig, grid, status, recLbl })
+ f.Controls.Add(c);
+
+ void Layout()
+ {
+ int W = f.ClientSize.Width, H = f.ClientSize.Height;
+ int X(double a) => (int)(W * a); int Y(double a) => (int)(H * a);
+ Rectangle R(double x0, double y0, double x1, double y1) => Rectangle.FromLTRB(X(x0), Y(y0), X(x1), Y(y1));
+ menu.Bounds = R(0, 0, 1, 0.045);
+ tool.Bounds = R(0, 0.045, 1, 0.095);
+ double lc = 0.13; // label/control split
+ nameLbl.Bounds = R(0, 0.105, lc, 0.16); nameBox.Bounds = R(lc, 0.105, 0.42, 0.16);
+ typeLbl.Bounds = R(0, 0.165, lc, 0.22); typeCmb.Bounds = R(lc, 0.167, 0.42, 0.22);
+ regionLbl.Bounds = R(0, 0.225, lc, 0.28); regionCmb.Bounds = R(lc, 0.227, 0.42, 0.28);
+ prioLbl.Bounds = R(0, 0.285, lc, 0.345); prio.Bounds = R(lc, 0.285, 0.42, 0.345);
+ creditLbl.Bounds = R(0, 0.35, lc, 0.41); credit.Bounds = R(lc, 0.35, 0.42, 0.41);
+ saveBtn.Bounds = R(0.04, 0.45, 0.40, 0.52);
+ sigHdr.Bounds = R(0.42, 0.095, 1, 0.135);
+ sig.Bounds = R(0.42, 0.135, 0.995, 0.42);
+ grid.Bounds = R(0.005, 0.55, 0.995, 0.93);
+ status.Bounds = R(0, 0.94, 0.5, 1);
+ recLbl.Bounds = R(0.5, 0.94, 1, 1);
+ float gfs = Math.Max(8, H / 70f);
+ grid.Font = new Font("Segoe UI", gfs);
+ grid.ColumnHeadersDefaultCellStyle.Font = new Font("Segoe UI", gfs, FontStyle.Bold);
+ }
+ f.Load += (s, e) => Layout();
+ f.Resize += (s, e) => Layout();
+ Application.Run(f);
+ }
+}
diff --git a/demo/multi-cursor/dotnet/winforms/winforms.csproj b/demo/multi-cursor/dotnet/winforms/winforms.csproj
new file mode 100644
index 0000000000..5b6864dd2d
--- /dev/null
+++ b/demo/multi-cursor/dotnet/winforms/winforms.csproj
@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+ <PropertyGroup>
+ <OutputType>WinExe</OutputType>
+ <TargetFramework>net10.0-windows</TargetFramework>
+ <UseWindowsForms>true</UseWindowsForms>
+ <Nullable>disable</Nullable>
+ <ImplicitUsings>disable</ImplicitUsings>
+ <AssemblyName>winforms-legacy</AssemblyName>
+ <ApplicationHighDpiMode>PerMonitorV2</ApplicationHighDpiMode>
+ </PropertyGroup>
+</Project>
diff --git a/demo/multi-cursor/dotnet/wpf/Program.cs b/demo/multi-cursor/dotnet/wpf/Program.cs
new file mode 100644
index 0000000000..7a5f07348e
--- /dev/null
+++ b/demo/multi-cursor/dotnet/wpf/Program.cs
@@ -0,0 +1,131 @@
+using System;
+using System.Collections.ObjectModel;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Controls.Primitives;
+using System.Windows.Media;
+using System.Windows.Shapes;
+
+// "Meridian CRM β€” Account Record" : WPF node (XAML / UIA). Real Slider /
+// ComboBox / InkCanvas (doodle) / DataGrid (spreadsheet). Account Name TextBox
+// + SAVE button are what cua-driver drives.
+class Program
+{
+ public class RowVM
+ {
+ public string Account { get; set; } public string Type { get; set; }
+ public string Region { get; set; } public int Pri { get; set; } public string Credit { get; set; }
+ }
+
+ [STAThread]
+ static void Main()
+ {
+ var app = new Application();
+ var rows = new ObservableCollection<RowVM>();
+ var gray = new SolidColorBrush(Color.FromRgb(0xE0, 0xE0, 0xE0));
+
+ // menu
+ var menu = new Menu();
+ foreach (var m in new[] { "File", "Edit", "View", "Record", "Tools", "Help" })
+ menu.Items.Add(new MenuItem { Header = m });
+ // toolbar
+ var tray = new ToolBarTray();
+ var tb = new ToolBar();
+ foreach (var t in new[] { "New", "Open", "Save", "Delete", "|", "β—€ Prev", "Next β–Ά", "|", "Find" })
+ tb.Items.Add(t == "|" ? (object)new Separator() : new Button { Content = t, Padding = new Thickness(6, 1, 6, 1) });
+ tray.ToolBars.Add(tb);
+
+ // ── left form (dense grid; cell borders only) ───────────────────────
+ var form = new Grid { Background = Brushes.White };
+ form.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(120) });
+ form.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(1, GridUnitType.Star) });
+ TextBlock lbl(string t) => new TextBlock { Text = " " + t, VerticalAlignment = VerticalAlignment.Center, Background = new SolidColorBrush(Color.FromRgb(0xF0, 0xF0, 0xF0)) };
+ int r = 0;
+ void AddRow(string label, FrameworkElement c, double h = 30)
+ {
+ form.RowDefinitions.Add(new RowDefinition { Height = new GridLength(h) });
+ var b1 = new Border { BorderBrush = gray, BorderThickness = new Thickness(0, 0, 1, 1), Child = lbl(label) };
+ Grid.SetRow(b1, r); Grid.SetColumn(b1, 0); form.Children.Add(b1);
+ var b2 = new Border { BorderBrush = gray, BorderThickness = new Thickness(0, 0, 0, 1), Child = c };
+ Grid.SetRow(b2, r); Grid.SetColumn(b2, 1); form.Children.Add(b2);
+ r++;
+ }
+ var nameBox = new TextBox { BorderThickness = new Thickness(0), VerticalContentAlignment = VerticalAlignment.Center };
+ var typeCmb = new ComboBox { BorderThickness = new Thickness(0) };
+ foreach (var s in new[] { "Enterprise", "SMB", "Government", "Reseller" }) typeCmb.Items.Add(s); typeCmb.SelectedIndex = 0;
+ var regionCmb = new ComboBox { BorderThickness = new Thickness(0) };
+ foreach (var s in new[] { "North", "South", "EMEA", "APAC", "LATAM" }) regionCmb.Items.Add(s); regionCmb.SelectedIndex = 0;
+ var priority = new Slider { Minimum = 1, Maximum = 5, Value = 3, TickFrequency = 1, IsSnapToTickEnabled = true, TickPlacement = TickPlacement.BottomRight, VerticalAlignment = VerticalAlignment.Center };
+ var credit = new Slider { Minimum = 0, Maximum = 100, Value = 40, TickFrequency = 10, TickPlacement = TickPlacement.BottomRight, VerticalAlignment = VerticalAlignment.Center };
+ AddRow("Account Name", nameBox);
+ AddRow("Account Type", typeCmb);
+ AddRow("Region", regionCmb);
+ AddRow("Priority (1-5)", priority);
+ AddRow("Credit Limit", credit);
+ var saveBtn = new Button { Content = "Add Record", FontWeight = FontWeights.Bold };
+ form.RowDefinitions.Add(new RowDefinition { Height = new GridLength(40) });
+ Grid.SetRow(saveBtn, r); Grid.SetColumn(saveBtn, 0); Grid.SetColumnSpan(saveBtn, 2); form.Children.Add(saveBtn);
+
+ // ── right top: line-tool sketch (one straight segment per drag) ─────
+ var ink = new Canvas { Background = Brushes.White, ClipToBounds = true };
+ Point? dn = null; Line preview = null;
+ ink.MouseLeftButtonDown += (s, e) => { dn = e.GetPosition(ink); };
+ ink.MouseMove += (s, e) => { if (dn != null) { var p = e.GetPosition(ink);
+ if (preview == null) { preview = new Line { Stroke = Brushes.MidnightBlue, StrokeThickness = 1, Opacity = 0.45 }; ink.Children.Add(preview); }
+ preview.X1 = dn.Value.X; preview.Y1 = dn.Value.Y; preview.X2 = p.X; preview.Y2 = p.Y; } };
+ ink.MouseLeftButtonUp += (s, e) => { if (dn != null) { var p = e.GetPosition(ink);
+ ink.Children.Add(new Line { Stroke = Brushes.MidnightBlue, StrokeThickness = 2, X1 = dn.Value.X, Y1 = dn.Value.Y, X2 = p.X, Y2 = p.Y });
+ if (preview != null) { ink.Children.Remove(preview); preview = null; }
+ dn = null; } };
+ var sigHdr = new TextBlock { Text = " Signature / Notes", FontWeight = FontWeights.Bold, Background = new SolidColorBrush(Color.FromRgb(0xE1,0xE1,0xE1)), Padding = new Thickness(2) };
+ var sigDock = new DockPanel(); DockPanel.SetDock(sigHdr, Dock.Top); sigDock.Children.Add(sigHdr); sigDock.Children.Add(ink);
+
+ // ── right bottom: spreadsheet ───────────────────────────────────────
+ var dg = new DataGrid { AutoGenerateColumns = false, ItemsSource = rows, IsReadOnly = true, GridLinesVisibility = DataGridGridLinesVisibility.All, HeadersVisibility = DataGridHeadersVisibility.Column };
+ dg.Columns.Add(new DataGridTextColumn { Header = "Account", Binding = new System.Windows.Data.Binding("Account"), Width = new DataGridLength(1, DataGridLengthUnitType.Star) });
+ dg.Columns.Add(new DataGridTextColumn { Header = "Type", Binding = new System.Windows.Data.Binding("Type") });
+ dg.Columns.Add(new DataGridTextColumn { Header = "Region", Binding = new System.Windows.Data.Binding("Region") });
+ dg.Columns.Add(new DataGridTextColumn { Header = "Pri", Binding = new System.Windows.Data.Binding("Pri") });
+ dg.Columns.Add(new DataGridTextColumn { Header = "Credit", Binding = new System.Windows.Data.Binding("Credit") });
+
+ var rightGrid = new Grid();
+ rightGrid.RowDefinitions.Add(new RowDefinition { Height = new GridLength(150) });
+ rightGrid.RowDefinitions.Add(new RowDefinition { Height = new GridLength(3) });
+ rightGrid.RowDefinitions.Add(new RowDefinition { Height = new GridLength(1, GridUnitType.Star) });
+ Grid.SetRow(sigDock, 0); rightGrid.Children.Add(sigDock);
+ var gs2 = new GridSplitter { Height = 3, HorizontalAlignment = HorizontalAlignment.Stretch, Background = gray }; Grid.SetRow(gs2, 1); rightGrid.Children.Add(gs2);
+ Grid.SetRow(dg, 2); rightGrid.Children.Add(dg);
+
+ var main = new Grid();
+ main.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(380) });
+ main.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(3) });
+ main.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(1, GridUnitType.Star) });
+ Grid.SetColumn(form, 0); main.Children.Add(form);
+ var gs1 = new GridSplitter { Width = 3, Background = gray }; Grid.SetColumn(gs1, 1); main.Children.Add(gs1);
+ Grid.SetColumn(rightGrid, 2); main.Children.Add(rightGrid);
+
+ // status bar
+ var statusBar = new StatusBar();
+ var statusLbl = new StatusBarItem { Content = "Ready" };
+ var recLbl = new StatusBarItem { Content = "Records: 0", HorizontalAlignment = HorizontalAlignment.Right };
+ statusBar.Items.Add(statusLbl);
+ statusBar.Items.Add(new StatusBarItem { Content = "USER: SYSTEM β–Œ CONNECTED", HorizontalAlignment = HorizontalAlignment.Right });
+ statusBar.Items.Add(recLbl);
+
+ int records = 0;
+ saveBtn.Click += (s, e) =>
+ {
+ string nm = (nameBox.Text ?? "").Trim(); if (nm.Length == 0) nm = "(unnamed)";
+ rows.Add(new RowVM { Account = nm, Type = typeCmb.Text, Region = regionCmb.Text, Pri = (int)priority.Value, Credit = "$" + ((int)credit.Value * 1000) });
+ records++; recLbl.Content = $"Records: {records}"; statusLbl.Content = $"Saved account '{nm}'.";
+ nameBox.Text = "";
+ };
+
+ var dock = new DockPanel();
+ DockPanel.SetDock(menu, Dock.Top); DockPanel.SetDock(tray, Dock.Top); DockPanel.SetDock(statusBar, Dock.Bottom);
+ dock.Children.Add(menu); dock.Children.Add(tray); dock.Children.Add(statusBar); dock.Children.Add(main);
+
+ var win = new Window { Title = "Meridian CRM β€” Account Record [WPF]", Width = 960, Height = 600, Content = dock, WindowStartupLocation = WindowStartupLocation.Manual };
+ app.Run(win);
+ }
+}
diff --git a/demo/multi-cursor/dotnet/wpf/wpf.csproj b/demo/multi-cursor/dotnet/wpf/wpf.csproj
new file mode 100644
index 0000000000..b2b65b75df
--- /dev/null
+++ b/demo/multi-cursor/dotnet/wpf/wpf.csproj
@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+ <PropertyGroup>
+ <OutputType>WinExe</OutputType>
+ <TargetFramework>net10.0-windows</TargetFramework>
+ <UseWPF>true</UseWPF>
+ <Nullable>disable</Nullable>
+ <ImplicitUsings>disable</ImplicitUsings>
+ <AssemblyName>wpf-legacy</AssemblyName>
+ <EnableDefaultApplicationDefinition>false</EnableDefaultApplicationDefinition>
+ </PropertyGroup>
+</Project>
diff --git a/demo/multi-cursor/electron/index.html b/demo/multi-cursor/electron/index.html
new file mode 100644
index 0000000000..1cb72ad276
--- /dev/null
+++ b/demo/multi-cursor/electron/index.html
@@ -0,0 +1,103 @@
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8" />
+<style>
+ * { box-sizing: border-box; margin: 0; padding: 0; }
+ html, body { height: 100%; font-family: "Segoe UI", Tahoma, sans-serif; font-size: 13px; background: #f0f0f0; overflow: hidden; }
+ #app { display: grid; grid-template-rows: auto auto 1fr auto; height: 100vh; }
+ /* menu + toolbar */
+ #menu { background: #f7f7f7; border-bottom: 1px solid #c8c8c8; padding: 2px 0; }
+ #menu span { padding: 3px 10px; }
+ #menu span:hover { background: #cde2ff; }
+ #toolbar { background: #eee; border-bottom: 1px solid #c8c8c8; display: flex; }
+ #toolbar button { border: 1px solid transparent; background: transparent; padding: 3px 9px; }
+ #toolbar button:hover { border: 1px solid #b0c4de; background: #e6eefb; }
+ #toolbar .sep { width: 1px; background: #c8c8c8; margin: 2px 3px; }
+ /* main split */
+ #main { display: grid; grid-template-columns: 380px 1px 1fr; overflow: hidden; }
+ #split { background: #c8c8c8; }
+ /* dense form: label | control rows, only cell borders */
+ #form { display: grid; grid-template-columns: 120px 1fr; grid-auto-rows: 30px; background: #fff; align-content: start; }
+ #form .lbl { background: #f0f0f0; border-right: 1px solid #e0e0e0; border-bottom: 1px solid #e0e0e0; display: flex; align-items: center; padding-left: 8px; }
+ #form .cell { border-bottom: 1px solid #e0e0e0; display: flex; align-items: center; }
+ #form .cell > * { width: 100%; height: 100%; border: 0; background: transparent; padding: 0 6px; font: inherit; }
+ #form input[type=range] { padding: 0; }
+ #saveRow { grid-column: 1 / span 2; }
+ #save { width: 100%; height: 38px; font-weight: bold; }
+ /* right side */
+ #right { display: grid; grid-template-rows: 150px 1px 1fr; overflow: hidden; }
+ #sigHdr { background: #e1e1e1; font-weight: bold; padding: 3px 6px; border-bottom: 1px solid #c8c8c8; }
+ #sigWrap { display: grid; grid-template-rows: auto 1fr; }
+ #doodle { width: 100%; height: 100%; background: #fff; display: block; }
+ table { border-collapse: collapse; width: 100%; background: #fff; }
+ th, td { border: 1px solid #d2d2d2; padding: 2px 6px; text-align: left; font-weight: normal; }
+ th { background: #eef1f5; font-weight: 600; }
+ #gridWrap { overflow: auto; }
+ #status { background: #e8e8e8; border-top: 1px solid #c8c8c8; display: flex; justify-content: space-between; padding: 3px 8px; }
+</style>
+</head>
+<body>
+ <div id="app">
+ <div id="menu"><span>File</span><span>Edit</span><span>View</span><span>Record</span><span>Tools</span><span>Help</span></div>
+ <div id="toolbar">
+ <button>New</button><button>Open</button><button>Save</button><button>Delete</button>
+ <div class="sep"></div><button>β—€ Prev</button><button>Next β–Ά</button>
+ <div class="sep"></div><button>Find</button>
+ </div>
+ <div id="main">
+ <div id="form">
+ <div class="lbl">Account Name</div><div class="cell"><input id="acct" type="text" /></div>
+ <div class="lbl">Account Type</div><div class="cell"><select id="type"><option>Enterprise</option><option>SMB</option><option>Government</option><option>Reseller</option></select></div>
+ <div class="lbl">Region</div><div class="cell"><select id="region"><option>North</option><option>South</option><option>EMEA</option><option>APAC</option><option>LATAM</option></select></div>
+ <div class="lbl">Priority (1-5)</div><div class="cell"><input id="pri" type="range" min="1" max="5" value="3" /></div>
+ <div class="lbl">Credit Limit</div><div class="cell"><input id="credit" type="range" min="0" max="100" value="40" /></div>
+ <div id="saveRow" class="cell"><button id="save">Add Record</button></div>
+ </div>
+ <div id="split"></div>
+ <div id="right">
+ <div id="sigWrap"><div id="sigHdr">Signature / Notes</div><canvas id="doodle"></canvas></div>
+ <div id="split"></div>
+ <div id="gridWrap">
+ <table id="grid"><thead><tr><th>Account</th><th>Type</th><th>Region</th><th>Pri</th><th>Credit</th></tr></thead><tbody></tbody></table>
+ </div>
+ </div>
+ </div>
+ <div id="status"><span id="statusMsg">Ready</span><span>USER: SYSTEM β–Œ CONNECTED</span><span id="recCount">Records: 0</span></div>
+ </div>
+ <script>
+ // line tool: one straight segment per press-drag-release (down→up)
+ const cv = document.getElementById('doodle');
+ const ctx = cv.getContext('2d');
+ let segs = [], dn = null, cur = null;
+ function redraw() {
+ ctx.clearRect(0, 0, cv.width, cv.height);
+ ctx.strokeStyle = '#191970'; ctx.lineWidth = 2;
+ for (const s of segs) { ctx.beginPath(); ctx.moveTo(s.x0, s.y0); ctx.lineTo(s.x1, s.y1); ctx.stroke(); }
+ if (dn && cur) { ctx.save(); ctx.globalAlpha = 0.45; ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(dn.x, dn.y); ctx.lineTo(cur.x, cur.y); ctx.stroke(); ctx.restore(); }
+ }
+ function fit() { cv.width = cv.clientWidth; cv.height = cv.clientHeight; redraw(); }
+ window.addEventListener('resize', fit); requestAnimationFrame(fit);
+ // Pointer events (not mouse): coordinate-routed background injection arrives
+ // as PEN/touch pointers; pointer* covers mouse + pen + touch uniformly.
+ cv.style.touchAction = 'none';
+ cv.addEventListener('pointerdown', e => { dn = { x: e.offsetX, y: e.offsetY }; cur = dn; });
+ cv.addEventListener('pointermove', e => { if (dn) { cur = { x: e.offsetX, y: e.offsetY }; redraw(); } });
+ cv.addEventListener('pointerup', e => { if (dn) { segs.push({ x0: dn.x, y0: dn.y, x1: e.offsetX, y1: e.offsetY }); dn = null; cur = null; redraw(); } });
+ // save
+ let records = 0;
+ const acct = document.getElementById('acct'), tbody = document.querySelector('#grid tbody');
+ document.getElementById('save').addEventListener('click', () => {
+ let nm = (acct.value || '').trim(); if (!nm) nm = '(unnamed)';
+ const tr = document.createElement('tr');
+ const credit = '$' + (document.getElementById('credit').value * 1000);
+ [nm, document.getElementById('type').value, document.getElementById('region').value, document.getElementById('pri').value, credit]
+ .forEach(v => { const td = document.createElement('td'); td.textContent = v; tr.appendChild(td); });
+ tbody.appendChild(tr);
+ records++; document.getElementById('recCount').textContent = `Records: ${records}`;
+ document.getElementById('statusMsg').textContent = `Saved account '${nm}'.`;
+ acct.value = '';
+ });
+ </script>
+</body>
+</html>
diff --git a/demo/multi-cursor/electron/main.js b/demo/multi-cursor/electron/main.js
new file mode 100644
index 0000000000..f8ef8da10e
--- /dev/null
+++ b/demo/multi-cursor/electron/main.js
@@ -0,0 +1,24 @@
+const { app, BrowserWindow } = require('electron');
+const path = require('path');
+
+function createWindow() {
+ const win = new BrowserWindow({
+ width: 900,
+ height: 560,
+ resizable: true,
+ title: 'Meridian CRM β€” Account Record [Electron]',
+ autoHideMenuBar: true,
+ webPreferences: { contextIsolation: true },
+ });
+ win.setMenuBarVisibility(false);
+ win.loadFile(path.join(__dirname, 'index.html'));
+}
+
+app.whenReady().then(() => {
+ createWindow();
+ app.on('activate', () => {
+ if (BrowserWindow.getAllWindows().length === 0) createWindow();
+ });
+});
+
+app.on('window-all-closed', () => app.quit());
diff --git a/demo/multi-cursor/electron/package.json b/demo/multi-cursor/electron/package.json
new file mode 100644
index 0000000000..0c36621cbf
--- /dev/null
+++ b/demo/multi-cursor/electron/package.json
@@ -0,0 +1,12 @@
+{
+ "name": "cua-demo-electron-legacy",
+ "version": "0.1.0",
+ "private": true,
+ "main": "main.js",
+ "scripts": {
+ "start": "electron ."
+ },
+ "devDependencies": {
+ "electron": "^33.0.0"
+ }
+}
diff --git a/demo/multi-cursor/legacy-app/Cargo.toml b/demo/multi-cursor/legacy-app/Cargo.toml
new file mode 100644
index 0000000000..59453e0f8d
--- /dev/null
+++ b/demo/multi-cursor/legacy-app/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "legacy-app"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "legacy-app"
+path = "src/main.rs"
+
+[dependencies]
+windows = { version = "0.58", features = [
+ "Win32_Foundation",
+ "Win32_UI_WindowsAndMessaging",
+ "Win32_Graphics_Gdi",
+ "Win32_System_LibraryLoader",
+] }
diff --git a/demo/multi-cursor/legacy-app/src/main.rs b/demo/multi-cursor/legacy-app/src/main.rs
new file mode 100644
index 0000000000..8960a97458
--- /dev/null
+++ b/demo/multi-cursor/legacy-app/src/main.rs
@@ -0,0 +1,343 @@
+//! "Meridian CRM β€” Account Record" β€” a dense, realistic Windows office form for
+//! the cua-driver multi-cursor demo. Menu bar, toolbar, a packed left form
+//! (Account Name, Account Type / Region dropdowns, Priority / Credit sliders),
+//! a SAVE button, a signature doodle pad, a spreadsheet grid, and a status bar.
+//! No padding between elements beyond their 1px borders.
+//!
+//! Modes (argv[1]):
+//! gdi <node> β€” fully GDI-drawn (NO accessibility tree β†’ cua-driver drives
+//! it via the pixel path). The Account-Name field and SAVE
+//! button are drawn; the doodle pad is live.
+//! master <node> β€” same chrome, but Account Name + SAVE are real Win32
+//! controls, instrumented to EMIT the user's action on stdout
+//! (TYPE<text>, CLICK<rx><ry>) for the orchestrator to replay.
+
+#![windows_subsystem = "windows"]
+
+use std::cell::RefCell;
+use std::io::Write;
+
+use windows::core::{w, PCWSTR};
+use windows::Win32::Foundation::{COLORREF, HINSTANCE, HWND, LPARAM, LRESULT, POINT, RECT, WPARAM};
+use windows::Win32::Graphics::Gdi::{
+ BeginPaint, CreateFontW, CreatePen, CreateSolidBrush, DeleteObject, DrawTextW, EndPaint, FillRect,
+ FrameRect, InvalidateRect, LineTo, MoveToEx, SelectObject, SetBkMode, SetTextColor, DT_CENTER, DT_LEFT,
+ DT_SINGLELINE, DT_VCENTER, GetStockObject, HBRUSH, HFONT, PAINTSTRUCT, PS_SOLID, TRANSPARENT, BLACK_BRUSH,
+};
+use windows::Win32::System::LibraryLoader::GetModuleHandleW;
+use windows::Win32::UI::WindowsAndMessaging::*;
+
+// ── dense fractional layout (shared by gdi + master) ──────────────────────────
+const MENU_B: f64 = 0.05;
+const TOOL_B: f64 = 0.10;
+// left form rows
+const NAME: [f64; 4] = [0.16, 0.110, 0.41, 0.170]; // Account Name field box
+const TYPE: [f64; 4] = [0.16, 0.175, 0.41, 0.235];
+const REGION: [f64; 4] = [0.16, 0.240, 0.41, 0.300];
+const PRIO: [f64; 4] = [0.16, 0.305, 0.41, 0.365];
+const CREDIT: [f64; 4] = [0.16, 0.370, 0.41, 0.430];
+const SAVE: [f64; 4] = [0.04, 0.460, 0.40, 0.530]; // SAVE button
+const DOODLE: [f64; 4] = [0.42, 0.140, 0.99, 0.420];
+const GRID: [f64; 4] = [0.01, 0.560, 0.99, 0.930];
+const STATUS_T: f64 = 0.94;
+// SAVE button relative center (master emits this for the CLICK event).
+const SAVE_CX: f64 = (SAVE[0] + SAVE[2]) / 2.0;
+const SAVE_CY: f64 = (SAVE[1] + SAVE[3]) / 2.0;
+
+const ID_EDIT: isize = 1001;
+const ID_BUTTON: isize = 1002;
+
+#[derive(Clone, Copy, PartialEq)]
+enum Mode { Gdi, Master }
+
+struct State {
+ mode: Mode,
+ node: String,
+ records: Vec<[String; 5]>,
+ name: String,
+ // line tool: one straight segment per press-drag-release (down→up)
+ segs: Vec<(POINT, POINT)>,
+ down: Option<POINT>,
+ cur: POINT,
+ hedit: HWND,
+ hbtn: HWND,
+ f_title: HFONT,
+ f_label: HFONT,
+ f_mono: HFONT,
+ cw: i32,
+ ch: i32,
+}
+
+thread_local! { static STATE: RefCell<Option<State>> = const { RefCell::new(None) }; }
+
+fn emit(line: &str) { let _ = writeln!(std::io::stdout(), "{line}"); let _ = std::io::stdout().flush(); }
+fn wide(s: &str) -> Vec<u16> { s.encode_utf16().chain(std::iter::once(0)).collect() }
+fn fr(cw: i32, ch: i32, r: [f64; 4]) -> RECT {
+ RECT { left: (cw as f64 * r[0]) as i32, top: (ch as f64 * r[1]) as i32,
+ right: (cw as f64 * r[2]) as i32, bottom: (ch as f64 * r[3]) as i32 }
+}
+fn band(cw: i32, ch: i32, y0: f64, y1: f64) -> RECT {
+ RECT { left: 0, top: (ch as f64 * y0) as i32, right: cw, bottom: (ch as f64 * y1) as i32 }
+}
+fn pt_in(r: &RECT, x: i32, y: i32) -> bool { x >= r.left && x < r.right && y >= r.top && y < r.bottom }
+unsafe fn mk_font(h: i32, w: i32, face: PCWSTR) -> HFONT { CreateFontW(h, 0, 0, 0, w, 0, 0, 0, 0, 0, 0, 0, 0, face) }
+
+fn main() {
+ let args: Vec<String> = std::env::args().collect();
+ let mode = if args.get(1).map(|s| s.as_str()) == Some("master") { Mode::Master } else { Mode::Gdi };
+ let node = args.get(2).cloned().unwrap_or_else(|| match mode { Mode::Master => "Master".into(), Mode::Gdi => "Win32 GDI".into() });
+
+ unsafe {
+ let hmod = GetModuleHandleW(None).unwrap();
+ let class = w!("MeridianCrmForm");
+ let bg = CreateSolidBrush(COLORREF(0x00F0F0F0));
+ let wc = WNDCLASSW { lpfnWndProc: Some(wnd_proc), hInstance: hmod.into(), lpszClassName: class,
+ hbrBackground: bg, hCursor: LoadCursorW(None, IDC_ARROW).unwrap_or_default(), ..Default::default() };
+ RegisterClassW(&wc);
+
+ STATE.with(|s| *s.borrow_mut() = Some(State { mode, node: node.clone(), records: Vec::new(),
+ name: String::new(), segs: Vec::new(), down: None, cur: POINT { x: 0, y: 0 }, hedit: HWND::default(), hbtn: HWND::default(),
+ f_title: HFONT::default(), f_label: HFONT::default(), f_mono: HFONT::default(), cw: 0, ch: 0 }));
+
+ let title = format!("Meridian CRM β€” Account Record [{node}]");
+ let tw = wide(&title);
+ let hwnd = CreateWindowExW(WINDOW_EX_STYLE(0), class, PCWSTR(tw.as_ptr()), WS_OVERLAPPEDWINDOW,
+ CW_USEDEFAULT, CW_USEDEFAULT, 960, 600, None, None, HINSTANCE(hmod.0), None).expect("CreateWindowExW");
+ let _ = ShowWindow(hwnd, SW_SHOWNORMAL);
+
+ let mut msg = MSG::default();
+ while GetMessageW(&mut msg, None, 0, 0).as_bool() { let _ = TranslateMessage(&msg); DispatchMessageW(&msg); }
+ }
+}
+
+unsafe fn create_master_controls(parent: HWND) {
+ let hinst = HINSTANCE(GetModuleHandleW(None).unwrap().0);
+ let hedit = CreateWindowExW(WS_EX_CLIENTEDGE, w!("EDIT"), w!(""),
+ WS_CHILD | WS_VISIBLE | WINDOW_STYLE(ES_AUTOHSCROLL as u32),
+ 0, 0, 10, 10, parent, HMENU(ID_EDIT as *mut core::ffi::c_void), hinst, None).unwrap_or_default();
+ let hbtn = CreateWindowExW(WINDOW_EX_STYLE(0), w!("BUTTON"), w!("Add Record"),
+ WS_CHILD | WS_VISIBLE | WINDOW_STYLE(BS_PUSHBUTTON as u32),
+ 0, 0, 10, 10, parent, HMENU(ID_BUTTON as *mut core::ffi::c_void), hinst, None).unwrap_or_default();
+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() { st.hedit = hedit; st.hbtn = hbtn; });
+}
+
+unsafe fn relayout(hwnd: HWND, cw: i32, ch: i32) {
+ STATE.with(|s| {
+ let mut b = s.borrow_mut(); let Some(st) = b.as_mut() else { return };
+ st.cw = cw; st.ch = ch;
+ for f in [st.f_title, st.f_label, st.f_mono] { if !f.is_invalid() { let _ = DeleteObject(f); } }
+ let seg = wide("Segoe UI"); let mono = wide("Consolas");
+ st.f_title = mk_font(-(ch / 34).clamp(12, 22), 700, PCWSTR(seg.as_ptr()));
+ st.f_label = mk_font(-(ch / 40).clamp(11, 18), 400, PCWSTR(seg.as_ptr()));
+ st.f_mono = mk_font(-(ch / 44).clamp(10, 16), 400, PCWSTR(mono.as_ptr()));
+ if st.mode == Mode::Master && !st.hedit.0.is_null() {
+ let n = fr(cw, ch, NAME);
+ let _ = MoveWindow(st.hedit, n.left + 2, n.top + 2, n.right - n.left - 4, n.bottom - n.top - 4, true);
+ let sv = fr(cw, ch, SAVE);
+ let _ = MoveWindow(st.hbtn, sv.left, sv.top, sv.right - sv.left, sv.bottom - sv.top, true);
+ SendMessageW(st.hedit, WM_SETFONT, WPARAM(st.f_label.0 as usize), LPARAM(1));
+ SendMessageW(st.hbtn, WM_SETFONT, WPARAM(st.f_label.0 as usize), LPARAM(1));
+ }
+ });
+ let _ = InvalidateRect(hwnd, None, true);
+}
+
+extern "system" fn wnd_proc(hwnd: HWND, msg: u32, wp: WPARAM, lp: LPARAM) -> LRESULT {
+ unsafe {
+ match msg {
+ WM_CREATE => { if STATE.with(|s| s.borrow().as_ref().map(|st| st.mode)) == Some(Mode::Master) { create_master_controls(hwnd); } LRESULT(0) }
+ WM_SIZE => { let (cw, ch) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32); if cw > 0 && ch > 0 { relayout(hwnd, cw, ch); } LRESULT(0) }
+ WM_COMMAND => { if (wp.0 & 0xFFFF) as isize == ID_BUTTON && ((wp.0 >> 16) & 0xFFFF) as u32 == BN_CLICKED { on_submit(hwnd); } LRESULT(0) }
+ WM_LBUTTONDOWN => {
+ let (x, y) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32);
+ let (mode, cw, ch) = STATE.with(|s| { let b = s.borrow(); let st = b.as_ref().unwrap(); (st.mode, st.cw, st.ch) });
+ if mode == Mode::Gdi && pt_in(&fr(cw, ch, SAVE), x, y) { commit(hwnd); }
+ else if pt_in(&fr(cw, ch, DOODLE), x, y) {
+ // line tool: remember the press point; commit on button-up.
+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() { st.down = Some(POINT { x, y }); st.cur = POINT { x, y }; });
+ }
+ LRESULT(0)
+ }
+ WM_MOUSEMOVE => {
+ if (wp.0 & 0x0001) != 0 {
+ let (x, y) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32);
+ let go = STATE.with(|s| { let mut b = s.borrow_mut(); let st = b.as_mut().unwrap();
+ if st.down.is_some() { st.cur = POINT { x, y }; true } else { false } });
+ if go { let (cw, ch) = STATE.with(|s| { let b = s.borrow(); (b.as_ref().unwrap().cw, b.as_ref().unwrap().ch) });
+ let d = fr(cw, ch, DOODLE); let _ = InvalidateRect(hwnd, Some(&d), false); }
+ }
+ LRESULT(0)
+ }
+ WM_LBUTTONUP => {
+ let (x, y) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32);
+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() {
+ if let Some(p0) = st.down.take() { st.segs.push((p0, POINT { x, y })); } });
+ let (cw, ch) = STATE.with(|s| { let b = s.borrow(); (b.as_ref().unwrap().cw, b.as_ref().unwrap().ch) });
+ let d = fr(cw, ch, DOODLE); let _ = InvalidateRect(hwnd, Some(&d), false);
+ LRESULT(0)
+ }
+ WM_CHAR => {
+ if STATE.with(|s| s.borrow().as_ref().map(|st| st.mode)) == Some(Mode::Gdi) {
+ let ch = char::from_u32(wp.0 as u32).unwrap_or('\0');
+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() {
+ match ch { '\u{8}' => { st.name.pop(); }, '\r' | '\n' => {}, c if !c.is_control() => st.name.push(c), _ => {} } });
+ let (cw, ch2) = STATE.with(|s| { let b = s.borrow(); (b.as_ref().unwrap().cw, b.as_ref().unwrap().ch) });
+ let n = fr(cw, ch2, NAME); let _ = InvalidateRect(hwnd, Some(&n), false);
+ }
+ LRESULT(0)
+ }
+ WM_PAINT => { paint(hwnd); LRESULT(0) }
+ WM_DESTROY => { PostQuitMessage(0); LRESULT(0) }
+ _ => DefWindowProcW(hwnd, msg, wp, lp),
+ }
+ }
+}
+
+unsafe fn on_submit(hwnd: HWND) {
+ let text = STATE.with(|s| { let b = s.borrow(); let st = b.as_ref()?; if st.hedit.0.is_null() { return None; }
+ let len = GetWindowTextLengthW(st.hedit); let mut buf = vec![0u16; (len + 1) as usize];
+ let n = GetWindowTextW(st.hedit, &mut buf); Some(String::from_utf16_lossy(&buf[..n as usize])) });
+ if let Some(t) = text {
+ emit(&format!("TYPE\t{t}"));
+ emit(&format!("CLICK\t{SAVE_CX:.4}\t{SAVE_CY:.4}"));
+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() { st.name = t; });
+ commit(hwnd);
+ STATE.with(|s| if let Some(st) = s.borrow().as_ref() { if !st.hedit.0.is_null() { let _ = SetWindowTextW(st.hedit, w!("")); } });
+ }
+}
+
+unsafe fn commit(hwnd: HWND) {
+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() {
+ let nm = if st.name.trim().is_empty() { "(unnamed)".into() } else { st.name.trim().chars().take(22).collect::<String>() };
+ st.records.push([nm, "Enterprise".into(), "North".into(), "3".into(), "$40,000".into()]);
+ st.name.clear();
+ });
+ let _ = InvalidateRect(hwnd, None, true);
+}
+
+unsafe fn line(hdc: windows::Win32::Graphics::Gdi::HDC, x: i32, y: i32, text: &str) {
+ let mut r = RECT { left: x, top: y, right: x + 6000, bottom: y + 60 }; let mut t = wide(text);
+ DrawTextW(hdc, &mut t, &mut r, DT_LEFT | DT_SINGLELINE);
+}
+
+unsafe fn paint(hwnd: HWND) {
+ let mut ps = PAINTSTRUCT::default();
+ let hdc = BeginPaint(hwnd, &mut ps);
+ SetBkMode(hdc, TRANSPARENT);
+ STATE.with(|s| {
+ let b = s.borrow(); let Some(st) = b.as_ref() else { return };
+ let (cw, ch) = (st.cw.max(1), st.ch.max(1));
+ let face = CreateSolidBrush(COLORREF(0x00F0F0F0));
+ let bar = CreateSolidBrush(COLORREF(0x00E8E8E8));
+ let hdr = CreateSolidBrush(COLORREF(0x00F5F1EE));
+ let white = CreateSolidBrush(COLORREF(0x00FFFFFF));
+ let line_c = CreateSolidBrush(COLORREF(0x00D2D2D2));
+ let labelbg = CreateSolidBrush(COLORREF(0x00F0F0F0));
+ let black = GetStockObject(BLACK_BRUSH);
+ let sel = CreateSolidBrush(COLORREF(0x00F5C28A)); // accent for slider thumb / save
+
+ let frame = |r: &RECT| { FrameRect(hdc, r, HBRUSH(black.0)); };
+ let hline = |y: i32| { let r = RECT { left: 0, top: y, right: cw, bottom: y + 1 }; FillRect(hdc, &r, line_c); };
+ let vline = |x: i32, y0: i32, y1: i32| { let r = RECT { left: x, top: y0, right: x + 1, bottom: y1 }; FillRect(hdc, &r, line_c); };
+
+ // menu bar
+ let mut m = band(cw, ch, 0.0, MENU_B); FillRect(hdc, &m, bar);
+ SelectObject(hdc, st.f_label); SetTextColor(hdc, COLORREF(0x00000000));
+ let mut mt = wide(" File Edit View Record Tools Help");
+ DrawTextW(hdc, &mut mt, &mut m, DT_LEFT | DT_VCENTER | DT_SINGLELINE);
+ hline((ch as f64 * MENU_B) as i32);
+ // toolbar
+ let mut tb = band(cw, ch, MENU_B, TOOL_B); FillRect(hdc, &tb, face);
+ let mut tt = wide(" New Open Save Delete β”‚ β—€ Prev Next β–Ά β”‚ Find");
+ DrawTextW(hdc, &mut tt, &mut tb, DT_LEFT | DT_VCENTER | DT_SINGLELINE);
+ hline((ch as f64 * TOOL_B) as i32);
+
+ // ── left form ──
+ let label = |r: [f64; 4], text: &str| {
+ let lr = RECT { left: (cw as f64 * 0.01) as i32, top: (ch as f64 * r[1]) as i32,
+ right: (cw as f64 * (r[0] - 0.005)) as i32, bottom: (ch as f64 * r[3]) as i32 };
+ FillRect(hdc, &lr, labelbg);
+ let mut t = wide(text);
+ let mut tr = RECT { left: lr.left + 6, ..lr };
+ DrawTextW(hdc, &mut t, &mut tr, DT_LEFT | DT_VCENTER | DT_SINGLELINE);
+ };
+ let dropdown = |r: [f64; 4], val: &str| {
+ let bx = fr(cw, ch, r); FillRect(hdc, &bx, white); frame(&bx);
+ let mut tr = bx; tr.left += 6; tr.right -= 24; let mut t = wide(val);
+ DrawTextW(hdc, &mut t, &mut tr, DT_LEFT | DT_VCENTER | DT_SINGLELINE);
+ let ar = RECT { left: bx.right - 20, top: bx.top, right: bx.right, bottom: bx.bottom };
+ FillRect(hdc, &ar, bar); frame(&ar); let mut a = wide("β–Ό");
+ DrawTextW(hdc, &mut a, &mut ar.clone(), DT_CENTER | DT_VCENTER | DT_SINGLELINE);
+ };
+ let slider = |r: [f64; 4], frac: f64| {
+ let bx = fr(cw, ch, r); let midy = (bx.top + bx.bottom) / 2;
+ let track = RECT { left: bx.left + 4, top: midy - 1, right: bx.right - 4, bottom: midy + 1 }; FillRect(hdc, &track, line_c);
+ let tx = bx.left + 4 + ((bx.right - bx.left - 8) as f64 * frac) as i32;
+ let thumb = RECT { left: tx - 4, top: bx.top + 6, right: tx + 4, bottom: bx.bottom - 6 }; FillRect(hdc, &thumb, sel); frame(&thumb);
+ };
+ SetTextColor(hdc, COLORREF(0x00000000));
+ label(NAME, "Account Name"); label(TYPE, "Account Type"); label(REGION, "Region");
+ label(PRIO, "Priority"); label(CREDIT, "Credit Limit");
+
+ if st.mode == Mode::Gdi {
+ let n = fr(cw, ch, NAME); FillRect(hdc, &n, white); frame(&n);
+ let mut tr = n; tr.left += 6; let mut nm = wide(&st.name);
+ DrawTextW(hdc, &mut nm, &mut tr, DT_LEFT | DT_VCENTER | DT_SINGLELINE);
+ let sv = fr(cw, ch, SAVE); FillRect(hdc, &sv, sel); frame(&sv);
+ let mut bt = wide("Add Record"); SelectObject(hdc, st.f_title);
+ DrawTextW(hdc, &mut bt, &mut sv.clone(), DT_CENTER | DT_VCENTER | DT_SINGLELINE);
+ SelectObject(hdc, st.f_label);
+ }
+ // dropdowns + sliders are visual in both modes
+ dropdown(TYPE, "Enterprise"); dropdown(REGION, "North");
+ slider(PRIO, 0.5); slider(CREDIT, 0.4);
+
+ // ── signature doodle ──
+ let dh = RECT { left: (cw as f64 * DOODLE[0]) as i32, top: (ch as f64 * 0.105) as i32,
+ right: (cw as f64 * DOODLE[2]) as i32, bottom: (ch as f64 * DOODLE[1]) as i32 };
+ FillRect(hdc, &dh, hdr); let mut sh = wide(" Signature / Notes");
+ SelectObject(hdc, st.f_title); DrawTextW(hdc, &mut sh, &mut dh.clone(), DT_LEFT | DT_VCENTER | DT_SINGLELINE);
+ SelectObject(hdc, st.f_label);
+ let dz = fr(cw, ch, DOODLE); FillRect(hdc, &dz, white); frame(&dz);
+ // line-tool segments (blue, 2px) + thin rubber-band preview while drawing
+ let pen = CreatePen(PS_SOLID, 2, COLORREF(0x00701919));
+ let old_pen = SelectObject(hdc, pen);
+ for (a, b) in &st.segs { let _ = MoveToEx(hdc, a.x, a.y, None); let _ = LineTo(hdc, b.x, b.y); }
+ if let Some(a) = st.down { let _ = MoveToEx(hdc, a.x, a.y, None); let _ = LineTo(hdc, st.cur.x, st.cur.y); }
+ SelectObject(hdc, old_pen); let _ = DeleteObject(pen);
+
+ // ── spreadsheet ──
+ let g = fr(cw, ch, GRID); FillRect(hdc, &g, white); frame(&g);
+ let cols = [("Account", 0.45), ("Type", 0.62), ("Region", 0.76), ("Pri", 0.85), ("Credit", 0.99)];
+ let cx = |fx: f64| (cw as f64 * (GRID[0] + (GRID[2] - GRID[0]) * ((fx - 0.0) / 1.0))) as i32; // placeholder
+ let _ = cx;
+ // column x positions in absolute (fractions are of full width but cols listed as cumulative within grid)
+ let gx = |fx: f64| (cw as f64 * fx) as i32;
+ let row_h = ((st.ch as f64 / 44.0).clamp(10.0, 16.0) * 1.7) as i32;
+ // header
+ let hrow = RECT { left: g.left, top: g.top, right: g.right, bottom: g.top + row_h }; FillRect(hdc, &hrow, hdr);
+ SelectObject(hdc, st.f_label);
+ let mut prev = g.left + 6;
+ for (name, fx) in cols { let mut t = wide(name); let mut tr = RECT { left: prev, top: g.top, right: gx(fx), bottom: g.top + row_h };
+ DrawTextW(hdc, &mut t, &mut tr, DT_LEFT | DT_VCENTER | DT_SINGLELINE); vline(gx(fx), g.top, g.bottom); prev = gx(fx) + 6; }
+ hline(g.top + row_h);
+ // rows
+ SelectObject(hdc, st.f_mono);
+ let mut y = g.top + row_h + 2;
+ for rec in &st.records {
+ if y + row_h > g.bottom { break; }
+ let xs = [g.left + 6, gx(0.45) + 6, gx(0.62) + 6, gx(0.76) + 6, gx(0.85) + 6];
+ for (i, val) in rec.iter().enumerate() { line(hdc, xs[i], y + 2, val); }
+ y += row_h; hline(y);
+ }
+
+ // ── status bar ──
+ let mut sb = band(cw, ch, STATUS_T, 1.0); FillRect(hdc, &sb, bar); hline((ch as f64 * STATUS_T) as i32);
+ SelectObject(hdc, st.f_label); SetTextColor(hdc, COLORREF(0x00000000));
+ let mut sbt = wide(&format!(" Ready β”‚ Records: {} β”‚ USER: SYSTEM β–Œ CONNECTED β”‚ [{}]", st.records.len(), st.node));
+ DrawTextW(hdc, &mut sbt, &mut sb, DT_LEFT | DT_VCENTER | DT_SINGLELINE);
+
+ for o in [face, bar, hdr, white, line_c, labelbg, sel] { let _ = DeleteObject(o); }
+ });
+ let _ = EndPaint(hwnd, &ps);
+}
diff --git a/demo/multi-cursor/orchestrator/Cargo.toml b/demo/multi-cursor/orchestrator/Cargo.toml
new file mode 100644
index 0000000000..2ee4227380
--- /dev/null
+++ b/demo/multi-cursor/orchestrator/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "orchestrator"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "orchestrator"
+path = "src/main.rs"
+
+[dependencies]
+# PNG decode + resize for the screenshot-based maze line-diff verification.
+image = { version = "0.25", default-features = false, features = ["png"] }
+windows = { version = "0.58", features = [
+ "Win32_Foundation",
+ "Win32_UI_WindowsAndMessaging",
+ "Win32_Graphics_Gdi",
+ "Win32_Graphics_Dwm",
+ "Win32_System_Threading",
+ "Win32_System_JobObjects",
+ "Win32_Security",
+] }
diff --git a/demo/multi-cursor/orchestrator/src/main.rs b/demo/multi-cursor/orchestrator/src/main.rs
new file mode 100644
index 0000000000..8c4cfd1209
--- /dev/null
+++ b/demo/multi-cursor/orchestrator/src/main.rs
@@ -0,0 +1,567 @@
+//! Multi-cursor background computer-use demo orchestrator.
+//!
+//! Launches the "National Records System" terminal in five UI frameworks: a 2x2
+//! grid of background windows (each = half the work area) around one foreground
+//! "master" in the middle that overlaps all four. When the human submits a
+//! record in the master, the master emits the action on stdout; this
+//! orchestrator replays it onto all four background corners concurrently, each
+//! via its OWN cua-driver session (= its own uniquely-coloured agent cursor),
+//! in the background β€” no window raised, the user's cursor never moved.
+//!
+//! Driving is element-based where an accessibility tree exists (set_value for
+//! the SUBJECT-NAME field + UIA-Invoke for SUBMIT β€” no foreground steal, no
+//! SendInput) and pixel-based for the GDI corner that has no a11y tree. Each
+//! a11y corner is VERIFIED by reading the records grid back: if the typed
+//! record didn't land, it's logged as FAIL.
+
+use std::io::{BufRead, BufReader};
+use std::path::PathBuf;
+use std::process::{Child, Command, Stdio};
+use std::sync::mpsc::{channel, Sender};
+use std::thread;
+use std::time::{Duration, Instant};
+
+use windows::core::PWSTR;
+use windows::Win32::Foundation::{BOOL, HANDLE, HWND, LPARAM, POINT, RECT, TRUE};
+use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS};
+use windows::Win32::Graphics::Gdi::ClientToScreen;
+use windows::Win32::System::JobObjects::{
+ AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject,
+ JobObjectExtendedLimitInformation, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
+ JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
+};
+use windows::Win32::UI::WindowsAndMessaging::{
+ EnumWindows, GetClientRect, GetWindowTextW, GetWindowThreadProcessId, IsWindowVisible,
+ SetForegroundWindow, SetWindowPos, HWND_TOP, SWP_NOACTIVATE, SWP_NOZORDER, SWP_SHOWWINDOW,
+};
+
+const FIELD_FRAC: (f64, f64) = (0.28, 0.145); // Account-Name field (GDI pixel fallback)
+
+// ── maze (line tool) ───────────────────────────────────────────────────────────
+// A spiral maze as straight segments in [0,1] of the drawing REGION. cua-driver
+// draws each as one press-drag-release (no curves = "line tool only"). The same
+// segments rasterized are the reference the screenshot is diffed against.
+const MAZE: [(f64, f64, f64, f64); 7] = [
+ (0.10, 0.12, 0.90, 0.12),
+ (0.90, 0.12, 0.90, 0.88),
+ (0.90, 0.88, 0.26, 0.88),
+ (0.26, 0.88, 0.26, 0.40),
+ (0.26, 0.40, 0.66, 0.40),
+ (0.66, 0.40, 0.66, 0.64),
+ (0.66, 0.64, 0.46, 0.64),
+];
+// Client-fraction rect of the window that sits INSIDE every framework's drawing
+// pad (all four put the doodle top-right). cua-driver draws here and the
+// verifier crops the same rect β€” so the rasterized reference aligns regardless
+// of where each framework actually lays its canvas out. Tuned via screenshots.
+// Kept inside the SHORTEST drawing pad: WPF/Electron host theirs in a fixed
+// ~150px row, so the usable band ends well above the spreadsheet below it.
+// Overshooting here drops the lower maze onto the grid (and selects rows).
+const REGION: [f64; 4] = [0.470, 0.180, 0.900, 0.295]; // l, t, r, b (client fractions)
+// A drawn maze must score at least this (F1 of ink overlap vs the reference,
+// dilated) to count as COMMITTED; below it = FAIL.
+const MAZE_PASS: f64 = 0.45;
+
+// ── kill-on-exit job ──────────────────────────────────────────────────────────
+static JOB: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
+fn job() -> HANDLE {
+ let raw = *JOB.get_or_init(|| unsafe {
+ let h = CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW");
+ let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
+ info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+ let _ = SetInformationJobObject(h, JobObjectExtendedLimitInformation,
+ &info as *const _ as *const core::ffi::c_void,
+ std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32);
+ h.0 as usize
+ });
+ HANDLE(raw as *mut core::ffi::c_void)
+}
+fn assign_to_job(child: &Child) {
+ use std::os::windows::io::AsRawHandle;
+ unsafe { let _ = AssignProcessToJobObject(job(), HANDLE(child.as_raw_handle() as *mut core::ffi::c_void)); }
+}
+
+struct Corner {
+ title: &'static str, // unique substring to find the window
+ session: &'static str, // palette name -> cursor color
+ hwnd: HWND,
+ pid: u32,
+ field_idx: Option<u64>, // UIA element for the SUBJECT-NAME field (a11y corners)
+ submit_idx: Option<u64>,// UIA element for the SUBMIT button
+}
+
+fn main() {
+ let demo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf();
+ let repo_root = demo_root.parent().unwrap().parent().unwrap().to_path_buf();
+
+ let cua = std::env::var("CUA_DRIVER_EXE").map(PathBuf::from)
+ .unwrap_or_else(|_| repo_root.join("libs/cua-driver/rust/target/debug/cua-driver.exe"));
+ let legacy_app = std::env::var("LEGACY_APP_EXE").map(PathBuf::from)
+ .unwrap_or_else(|_| demo_root.join("target/debug/legacy-app.exe"));
+ if !cua.exists() { eprintln!("cua-driver.exe not found at {cua:?}"); std::process::exit(1); }
+ if !legacy_app.exists() { eprintln!("legacy-app.exe not found at {legacy_app:?}"); std::process::exit(1); }
+
+ eprintln!("[orch] starting cua-driver daemon…");
+ let mut daemon = Command::new(&cua).arg("serve").stdout(Stdio::null()).stderr(Stdio::null())
+ .spawn().expect("spawn cua-driver serve");
+ assign_to_job(&daemon);
+ thread::sleep(Duration::from_millis(1500));
+
+ // Launch corners + master.
+ let mut kids: Vec<Child> = Vec::new();
+ let mut launch = |kids: &mut Vec<Child>, cmd: &mut Command| {
+ if let Ok(c) = cmd.stdout(Stdio::null()).stderr(Stdio::null()).spawn() { assign_to_job(&c); kids.push(c); }
+ };
+ launch(&mut kids, Command::new(&legacy_app).args(["gdi", "Win32 GDI (no a11y)"]));
+ let winforms = demo_root.join("dotnet/winforms/bin/Debug/net10.0-windows/winforms-legacy.exe");
+ if winforms.exists() { launch(&mut kids, &mut Command::new(&winforms)); } else { eprintln!("[orch] (skip) winforms missing"); }
+ let wpf = demo_root.join("dotnet/wpf/bin/Debug/net10.0-windows/wpf-legacy.exe");
+ if wpf.exists() { launch(&mut kids, &mut Command::new(&wpf)); } else { eprintln!("[orch] (skip) wpf missing"); }
+ let electron_bin = demo_root.join("electron/node_modules/.bin/electron.cmd");
+ if electron_bin.exists() {
+ let mut c = Command::new(&electron_bin); c.arg(".").current_dir(demo_root.join("electron"));
+ launch(&mut kids, &mut c);
+ } else { eprintln!("[orch] (skip) electron not installed"); }
+
+ let mut master = Command::new(&legacy_app).args(["master", "Master (Win32 controls)"])
+ .stdout(Stdio::piped()).stderr(Stdio::null()).spawn().expect("spawn master");
+ assign_to_job(&master);
+ let master_out = master.stdout.take().unwrap();
+
+ thread::sleep(Duration::from_secs(3)); // window + electron warmup
+
+ let mut corners = vec![
+ Corner { title: "Win32 GDI", session: "crimson", hwnd: HWND::default(), pid: 0, field_idx: None, submit_idx: None },
+ Corner { title: "WinForms", session: "amber", hwnd: HWND::default(), pid: 0, field_idx: None, submit_idx: None },
+ Corner { title: "WPF", session: "aqua", hwnd: HWND::default(), pid: 0, field_idx: None, submit_idx: None },
+ Corner { title: "Electron", session: "mint_lime", hwnd: HWND::default(), pid: 0, field_idx: None, submit_idx: None },
+ ];
+ for c in corners.iter_mut() {
+ if let Some((h, pid)) = find_window_by_title(c.title) { c.hwnd = h; c.pid = pid; }
+ else { eprintln!("[orch] (skip) no window for '{}'", c.title); }
+ }
+ corners.retain(|c| !c.hwnd.0.is_null());
+ let master_hwnd = find_window_by_title("Master (Win32").map(|(h, _)| h);
+
+ // Layout: each window half the work area; corners tile quadrants, master centered on top.
+ let wa = work_area();
+ let ww = (wa.right - wa.left) / 2;
+ let wh = (wa.bottom - wa.top) / 2;
+ let positions = [(wa.left, wa.top), (wa.left + ww, wa.top), (wa.left, wa.top + wh), (wa.left + ww, wa.top + wh)];
+ for (i, c) in corners.iter().enumerate() { let (x, y) = positions[i % 4]; place(c.hwnd, x, y, ww, wh, false); }
+ if let Some(mh) = master_hwnd {
+ // Master is smaller than a quadrant so it overlaps the central seam of
+ // all four corners without hiding their forms (which sit at the
+ // quadrant centers, outside this box). Centered on the work area.
+ let (mw, mh2) = (ww / 2, wh / 2);
+ let (mx, my) = (wa.left + ww - mw / 2, wa.top + wh - mh2 / 2);
+ place(mh, mx, my, mw, mh2, true);
+ unsafe { let _ = SetForegroundWindow(mh); }
+ }
+ thread::sleep(Duration::from_millis(800)); // let resized layouts settle
+
+ // Discover UIA elements per corner (field + SUBMIT). GDI has none -> pixel.
+ for c in corners.iter_mut() {
+ let tree = run_call_out(&cua, "get_window_state",
+ &format!(r#"{{"pid":{},"window_id":{},"capture_mode":"ax","session":"{}"}}"#, c.pid, c.hwnd.0 as isize, c.session));
+ c.field_idx = find_idx(&tree, |line| line.contains("] Edit"));
+ c.submit_idx = find_idx(&tree, |line| line.contains("] Button") && line.to_uppercase().contains("ADD RECORD"));
+ eprintln!("[orch] {:<9} session={:<9} field={:?} submit={:?} {}",
+ c.title, c.session, c.field_idx, c.submit_idx,
+ if c.field_idx.is_some() { "(a11y: set_value+invoke)" } else { "(no a11y: pixel)" });
+ // Pre-arm a coloured cursor for this session.
+ let _ = run_call(&cua, "set_agent_cursor_enabled", &format!(r#"{{"enabled":true,"session":"{}"}}"#, c.session));
+ }
+
+ // One driver thread per corner.
+ let mut senders: Vec<Sender<Action>> = Vec::new();
+ let mut handles = Vec::new();
+ for c in &corners {
+ let (tx, rx) = channel::<Action>();
+ senders.push(tx);
+ let cua = cua.clone();
+ let pid = c.pid;
+ let hwnd_addr = c.hwnd.0 as isize;
+ let session = c.session.to_string();
+ let title = c.title.to_string();
+ handles.push(thread::spawn(move || {
+ let hwnd = HWND(hwnd_addr as *mut _);
+ let ax = format!(r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"ax","session":"{session}"}}"#);
+ let mut seen_records: i64 = -1;
+ for act in rx {
+ // Re-discover element indices on a FRESH snapshot each action β€”
+ // a11y vs pixel is decided per action (Chromium's tree appears
+ // late and re-numbers as rows are added; GDI never has a tree).
+ let tree = run_call_out(&cua, "get_window_state", &ax);
+ let field = find_idx(&tree, |l| l.contains("] Edit"));
+ let submit = find_idx(&tree, |l| l.contains("] Button") && l.to_uppercase().contains("ADD RECORD"));
+ match act {
+ Action::Type { text } => {
+ let ok = match field {
+ // a11y: plain type_text with element_index β€” cua-driver
+ // auto-routes to UIA ValuePattern.SetValue on its own.
+ Some(idx) => run_call(&cua, "type_text", &format!(
+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"element_index":{idx},"text":"{}","session":"{session}"}}"#, json_escape(&text))),
+ // no a11y (GDI): focus the field by pixel, then WM_CHAR.
+ None => {
+ if let Some((x, y)) = client_rel_to_local_px(hwnd, FIELD_FRAC.0, FIELD_FRAC.1) {
+ let _ = run_call(&cua, "click", &format!(
+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"x":{x},"y":{y},"session":"{session}"}}"#));
+ }
+ run_call(&cua, "type_text", &format!(
+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"text":"{}","session":"{session}"}}"#, json_escape(&text)))
+ }
+ };
+ eprintln!("[drive {session}] type {text:?} -> {}", if ok { "ok" } else { "FAIL" });
+ }
+ Action::Click { rx, ry } => {
+ let ok = match submit {
+ Some(idx) => run_call(&cua, "click", &format!(
+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"element_index":{idx},"session":"{session}"}}"#)),
+ None => {
+ if let Some((x, y)) = client_rel_to_local_px(hwnd, rx, ry) {
+ run_call(&cua, "click", &format!(
+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"x":{x},"y":{y},"session":"{session}"}}"#))
+ } else { false }
+ }
+ };
+ eprintln!("[drive {session}] click Add Record -> {}", if ok { "ok" } else { "FAIL" });
+
+ // Verify a record was actually COMMITTED: the "Records: N"
+ // counter must increment. (Checking for the typed text
+ // alone is fooled by it sitting in the field.) Only the
+ // a11y corners expose the counter; GDI's is drawn pixels,
+ // so it's verified visually (its grid grows on screen).
+ thread::sleep(Duration::from_millis(300));
+ let t2 = run_call_out(&cua, "get_window_state", &ax);
+ if let Some(n) = records_count(&t2) {
+ let landed = n as i64 > seen_records;
+ seen_records = n as i64;
+ eprintln!("[verify {session}] {title} Records={n} -> {}",
+ if landed { "COMMITTED βœ“" } else { "FAIL (no new record)" });
+ }
+ }
+ Action::DrawMaze => {
+ // Pure coordinate drawing β€” NO element targeting, NO app
+ // cooperation. Each maze segment is one straight
+ // press-drag-release into the common REGION; cua-driver
+ // PostMessages the drag where it can and pen-injects it
+ // where the canvas (Chromium/WPF) drops posted mouse.
+ let map = |mx: f64, my: f64| {
+ let cfx = REGION[0] + mx * (REGION[2] - REGION[0]);
+ let cfy = REGION[1] + my * (REGION[3] - REGION[1]);
+ client_rel_to_local_px(hwnd, cfx, cfy)
+ };
+ let mut drawn = 0;
+ for (x0, y0, x1, y1) in MAZE.iter() {
+ if let (Some((fx, fy)), Some((tx, ty))) = (map(*x0, *y0), map(*x1, *y1)) {
+ if run_call(&cua, "drag", &format!(
+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"from_x":{fx},"from_y":{fy},"to_x":{tx},"to_y":{ty},"dispatch":"background","session":"{session}"}}"#)) {
+ drawn += 1;
+ }
+ }
+ thread::sleep(Duration::from_millis(120));
+ }
+ eprintln!("[draw {session}] maze: {drawn}/{} segments", MAZE.len());
+
+ // Read back by SCREENSHOT (WGC captures the occluded
+ // window) and diff the rendered ink against the
+ // rasterized reference maze β€” normalized into the same
+ // REGION grid, pass only under threshold.
+ thread::sleep(Duration::from_millis(400));
+ let shot = std::env::temp_dir().join(format!("maze_{session}.png"));
+ let _ = std::fs::remove_file(&shot);
+ run_call_shot(&cua, &format!(
+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"vision","max_image_dimension":4096,"session":"{session}"}}"#), &shot);
+ match score_maze(hwnd, &shot) {
+ Some(score) => eprintln!("[verify {session}] {title} maze score={score:.2} -> {}",
+ if score >= MAZE_PASS { "DREW MAZE βœ“" } else { "FAIL (lines don't match)" }),
+ None => eprintln!("[verify {session}] {title} maze -> FAIL (no screenshot/region)"),
+ }
+ }
+ }
+ }
+ }));
+ }
+
+ eprintln!("[orch] ready β€” submit a record in the center MASTER; watch {} coloured cursors \
+ drive the corner terminals in the background.", corners.len());
+
+ if std::env::args().any(|a| a == "--auto") {
+ let s2 = senders.clone();
+ thread::spawn(move || {
+ for i in 1..=3 {
+ thread::sleep(Duration::from_secs(3));
+ eprintln!("[orch] AUTO {i}: TYPE then CLICK");
+ for tx in &s2 { let _ = tx.send(Action::Type { text: format!("SUBJECT-{i:03}") }); }
+ thread::sleep(Duration::from_millis(1800));
+ // SAVE/"Add Record" button center (matches the GDI/master layout)
+ // for the pixel-driven GDI corner; a11y corners ignore rx/ry.
+ for tx in &s2 { let _ = tx.send(Action::Click { rx: 0.22, ry: 0.495 }); }
+ }
+ // Finally: draw the same maze into every corner's drawing pad by
+ // pure coordinate drags, then screenshot-diff each against the
+ // reference. Proves canvas actuation with no a11y, no app help.
+ thread::sleep(Duration::from_secs(2));
+ eprintln!("[orch] AUTO 4: DRAW MAZE (line tool) + screenshot diff");
+ for tx in &s2 { let _ = tx.send(Action::DrawMaze); }
+ });
+ }
+
+ let reader = BufReader::new(master_out);
+ for line in reader.lines().map_while(Result::ok) {
+ let parts: Vec<&str> = line.trim().split('\t').collect();
+ let action = match parts.as_slice() {
+ ["CLICK", rx, ry] => rx.parse::<f64>().ok().zip(ry.parse::<f64>().ok()).map(|(rx, ry)| Action::Click { rx, ry }),
+ ["TYPE", text] => Some(Action::Type { text: (*text).to_string() }),
+ _ => None,
+ };
+ if let Some(a) = action {
+ eprintln!("[orch] user action: {a:?} -> driving {} corners", senders.len());
+ for tx in &senders { let _ = tx.send(a.clone()); }
+ }
+ }
+
+ drop(senders);
+ for h in handles { let _ = h.join(); }
+ let _ = master.kill();
+ for mut k in kids { let _ = k.kill(); }
+ let _ = daemon.kill();
+}
+
+#[derive(Clone, Debug)]
+enum Action { Click { rx: f64, ry: f64 }, Type { text: String }, DrawMaze }
+
+fn json_escape(s: &str) -> String {
+ let mut o = String::with_capacity(s.len());
+ for ch in s.chars() {
+ match ch { '"' => o.push_str("\\\""), '\\' => o.push_str("\\\\"), '\n' => o.push_str("\\n"),
+ '\r' => {}, '\t' => o.push_str("\\t"), c => o.push(c) }
+ }
+ o
+}
+
+/// `cua-driver call <tool> <json>` (proxies to the daemon). True on success.
+fn run_call(cua: &PathBuf, tool: &str, json: &str) -> bool {
+ Command::new(cua).arg("call").arg(tool).arg(json)
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .status().map(|s| s.success()).unwrap_or(false)
+}
+/// Same, capturing stdout (for get_window_state readback).
+fn run_call_out(cua: &PathBuf, tool: &str, json: &str) -> String {
+ Command::new(cua).arg("call").arg(tool).arg(json)
+ .stderr(Stdio::null()).output()
+ .map(|o| String::from_utf8_lossy(&o.stdout).into_owned()).unwrap_or_default()
+}
+/// get_window_state writing the screenshot PNG to `out` (WGC; captures even a
+/// fully occluded window). Used to read back what cua-driver drew.
+fn run_call_shot(cua: &PathBuf, json: &str, out: &std::path::Path) -> bool {
+ Command::new(cua).arg("call").arg("get_window_state").arg(json)
+ .arg("--screenshot-out-file").arg(out)
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .status().map(|s| s.success()).unwrap_or(false)
+}
+
+/// DWM extended-frame size of `hwnd` β€” the pixel size of the WGC bitmap before
+/// get_window_state downscales it to fit `max_image_dimension`.
+fn window_frame_size(hwnd: HWND) -> Option<(i32, i32)> {
+ unsafe {
+ let mut d = RECT::default();
+ if DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &mut d as *mut _ as *mut core::ffi::c_void, std::mem::size_of::<RECT>() as u32).is_err() {
+ return None;
+ }
+ Some((d.right - d.left, d.bottom - d.top))
+ }
+}
+
+/// Shape-only F1 (0..1) of the maze cua-driver drew vs the reference maze.
+///
+/// Reads the ink from the window screenshot (cropped to the drawing REGION),
+/// then normalizes BOTH the drawn ink and the reference into a unit grid by
+/// their own bounding boxes β€” so the score reflects the *shape* of the lines,
+/// not where on the canvas they landed or at what scale (the user only wants
+/// the shape penalized). The screenshot is downscaled to fit
+/// `max_image_dimension`, so REGION (full-res bitmap px) is scaled by the
+/// PNG/frame ratio before cropping.
+fn score_maze(hwnd: HWND, png: &std::path::Path) -> Option<f64> {
+ let img = image::open(png).ok()?.to_luma8();
+ let (iw, ih) = (img.width() as i32, img.height() as i32);
+ let (fw, fh) = window_frame_size(hwnd)?;
+ if fw <= 0 || fh <= 0 { return None; }
+ let (fx, fy) = (iw as f64 / fw as f64, ih as f64 / fh as f64);
+ let (ax, ay) = client_rel_to_local_px(hwnd, REGION[0], REGION[1])?;
+ let (bx, by) = client_rel_to_local_px(hwnd, REGION[2], REGION[3])?;
+ let cx0 = (((ax.min(bx)) as f64 * fx).floor() as i32).max(0);
+ let cy0 = (((ay.min(by)) as f64 * fy).floor() as i32).max(0);
+ let cx1 = (((ax.max(bx)) as f64 * fx).ceil() as i32).min(iw - 1);
+ let cy1 = (((ay.max(by)) as f64 * fy).ceil() as i32).min(ih - 1);
+ if cx1 - cx0 < 12 || cy1 - cy0 < 12 { return None; }
+
+ // Drawn ink + its bounding box.
+ let mut ink: Vec<(i32, i32)> = Vec::new();
+ let (mut nx, mut ny, mut xx, mut xy) = (i32::MAX, i32::MAX, i32::MIN, i32::MIN);
+ for y in cy0..=cy1 {
+ for x in cx0..=cx1 {
+ if img.get_pixel(x as u32, y as u32).0[0] < 150 {
+ ink.push((x, y));
+ if x < nx { nx = x } if x > xx { xx = x }
+ if y < ny { ny = y } if y > xy { xy = y }
+ }
+ }
+ }
+ const N: usize = 96;
+ if ink.len() < 20 || xx <= nx || xy <= ny { return None; }
+ let (bw, bh) = ((xx - nx) as f64, (xy - ny) as f64);
+ let mut a = vec![false; N * N];
+ for (x, y) in ink {
+ let gx = (((x - nx) as f64 / bw) * (N as f64 - 1.0)).round() as usize;
+ let gy = (((y - ny) as f64 / bh) * (N as f64 - 1.0)).round() as usize;
+ a[gy.min(N - 1) * N + gx.min(N - 1)] = true;
+ }
+
+ // Reference maze, normalized to ITS bounding box the same way.
+ let (mut rnx, mut rny, mut rxx, mut rxy) = (f64::MAX, f64::MAX, f64::MIN, f64::MIN);
+ for (x0, y0, x1, y1) in MAZE.iter() {
+ for (px, py) in [(*x0, *y0), (*x1, *y1)] {
+ if px < rnx { rnx = px } if px > rxx { rxx = px }
+ if py < rny { rny = py } if py > rxy { rxy = py }
+ }
+ }
+ let (rbw, rbh) = ((rxx - rnx).max(1e-6), (rxy - rny).max(1e-6));
+ let mut b = vec![false; N * N];
+ for (x0, y0, x1, y1) in MAZE.iter() {
+ raster_line(&mut b, N, (x0 - rnx) / rbw, (y0 - rny) / rbh, (x1 - rnx) / rbw, (y1 - rny) / rbh);
+ }
+ Some(mask_f1(&a, &b, N, 2))
+}
+
+/// Bresenham rasterize a normalized [0,1] segment into the NΓ—N reference mask.
+fn raster_line(mask: &mut [bool], n: usize, x0: f64, y0: f64, x1: f64, y1: f64) {
+ let s = n as f64 - 1.0;
+ let (px1, py1) = ((x1 * s) as i32, (y1 * s) as i32);
+ let (mut x, mut y) = ((x0 * s) as i32, (y0 * s) as i32);
+ let (dx, dy) = ((px1 - x).abs(), -(py1 - y).abs());
+ let (sx, sy) = (if x < px1 { 1 } else { -1 }, if y < py1 { 1 } else { -1 });
+ let mut err = dx + dy;
+ loop {
+ if x >= 0 && y >= 0 && (x as usize) < n && (y as usize) < n { mask[y as usize * n + x as usize] = true; }
+ if x == px1 && y == py1 { break; }
+ let e2 = 2 * err;
+ if e2 >= dy { err += dy; x += sx; }
+ if e2 <= dx { err += dx; y += sy; }
+ }
+}
+
+/// Chebyshev dilation by radius `r`.
+fn dilate(m: &[bool], n: usize, r: i32) -> Vec<bool> {
+ let mut o = vec![false; n * n];
+ for y in 0..n as i32 {
+ for x in 0..n as i32 {
+ if !m[y as usize * n + x as usize] { continue; }
+ for dy in -r..=r {
+ for dx in -r..=r {
+ let (nx, ny) = (x + dx, y + dy);
+ if nx >= 0 && ny >= 0 && (nx as usize) < n && (ny as usize) < n { o[ny as usize * n + nx as usize] = true; }
+ }
+ }
+ }
+ }
+ o
+}
+
+/// F1 of two ink masks, each dilated by `r` so near-misses count as matches.
+fn mask_f1(a: &[bool], b: &[bool], n: usize, r: i32) -> f64 {
+ let (da, db) = (dilate(a, n, r), dilate(b, n, r));
+ let (mut ca, mut cb, mut ma, mut mb) = (0usize, 0usize, 0usize, 0usize);
+ for i in 0..n * n {
+ if a[i] { ca += 1; if db[i] { ma += 1; } }
+ if b[i] { cb += 1; if da[i] { mb += 1; } }
+ }
+ if ca == 0 || cb == 0 { return 0.0; }
+ let precision = ma as f64 / ca as f64; // drawn ink that is near the reference
+ let recall = mb as f64 / cb as f64; // reference covered by drawn ink
+ if precision + recall == 0.0 { 0.0 } else { 2.0 * precision * recall / (precision + recall) }
+}
+
+/// First `[N]` element index on a tree line matching `pred`.
+fn find_idx(tree: &str, pred: impl Fn(&str) -> bool) -> Option<u64> {
+ // The tree is one JSON line with literal "\n" separators between rows.
+ for line in tree.split("\\n") {
+ if !pred(line) { continue; }
+ let st = line.find('[')? + 1;
+ let en = line[st..].find(']')? + st;
+ if let Ok(n) = line[st..en].trim().parse() { return Some(n); }
+ }
+ None
+}
+
+/// Parse the "Records: N" counter from a window-state tree (status bar).
+fn records_count(tree: &str) -> Option<u32> {
+ let pos = tree.find("Records:")?;
+ let rest = &tree[pos + "Records:".len()..];
+ let digits: String = rest.trim_start().chars().take_while(|c| c.is_ascii_digit()).collect();
+ digits.parse().ok()
+}
+
+fn work_area() -> RECT {
+ use windows::Win32::UI::WindowsAndMessaging::{SystemParametersInfoW, SPI_GETWORKAREA, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS};
+ let mut wa = RECT::default();
+ unsafe { let _ = SystemParametersInfoW(SPI_GETWORKAREA, 0, Some(&mut wa as *mut _ as *mut core::ffi::c_void), SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0)); }
+ if wa.right <= wa.left || wa.bottom <= wa.top {
+ use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};
+ unsafe { wa = RECT { left: 0, top: 0, right: GetSystemMetrics(SM_CXSCREEN), bottom: GetSystemMetrics(SM_CYSCREEN) }; }
+ }
+ wa
+}
+
+fn place(hwnd: HWND, x: i32, y: i32, w: i32, h: i32, activate: bool) {
+ if hwnd.0.is_null() { return; }
+ let flags = if activate { SWP_SHOWWINDOW } else { SWP_NOACTIVATE | SWP_SHOWWINDOW | SWP_NOZORDER };
+ unsafe { let _ = SetWindowPos(hwnd, HWND_TOP, x, y, w, h, flags); }
+}
+
+/// Client-relative (0..1) -> the click tool's window-local screenshot-pixel
+/// space (ClientToScreen minus the DWM extended-frame top-left + 1px inset).
+fn client_rel_to_local_px(hwnd: HWND, rx: f64, ry: f64) -> Option<(i32, i32)> {
+ unsafe {
+ let mut cr = RECT::default();
+ GetClientRect(hwnd, &mut cr).ok()?;
+ let mut pt = POINT { x: (rx * (cr.right - cr.left) as f64) as i32, y: (ry * (cr.bottom - cr.top) as f64) as i32 };
+ let _ = ClientToScreen(hwnd, &mut pt);
+ let mut dwm = RECT::default();
+ if DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &mut dwm as *mut _ as *mut core::ffi::c_void, std::mem::size_of::<RECT>() as u32).is_err() {
+ return Some((pt.x, pt.y));
+ }
+ Some((pt.x - dwm.left - 1, pt.y - dwm.top - 1))
+ }
+}
+
+// ── window discovery by title substring (case-insensitive) ────────────────────
+struct Finder { needle: String, hwnd: HWND, pid: u32 }
+unsafe extern "system" fn enum_cb(hwnd: HWND, lparam: LPARAM) -> BOOL {
+ let f = &mut *(lparam.0 as *mut Finder);
+ if !IsWindowVisible(hwnd).as_bool() { return TRUE; }
+ let mut buf = [0u16; 256];
+ let n = GetWindowTextW(hwnd, &mut buf);
+ if n > 0 {
+ let title = String::from_utf16_lossy(&buf[..n as usize]);
+ if title.to_lowercase().contains(&f.needle.to_lowercase()) {
+ let mut pid = 0u32;
+ GetWindowThreadProcessId(hwnd, Some(&mut pid));
+ f.hwnd = hwnd; f.pid = pid;
+ return BOOL(0);
+ }
+ }
+ let _ = PWSTR::null();
+ TRUE
+}
+fn find_window_by_title(needle: &str) -> Option<(HWND, u32)> {
+ let deadline = Instant::now() + Duration::from_secs(8);
+ loop {
+ let mut f = Finder { needle: needle.to_string(), hwnd: HWND::default(), pid: 0 };
+ unsafe { let _ = EnumWindows(Some(enum_cb), LPARAM(&mut f as *mut _ as isize)); }
+ if !f.hwnd.0.is_null() { return Some((f.hwnd, f.pid)); }
+ if Instant::now() > deadline { return None; }
+ thread::sleep(Duration::from_millis(300));
+ }
+}
diff --git a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
index affa006490..57156b1f29 100644
--- a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
+++ b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml
@@ -69,4 +69,15 @@ tempfile = "3"
[target.'cfg(target_os = "windows")'.dev-dependencies]
platform-windows = { path = "../platform-windows" }
-windows = { version = "0.61", features = ["Win32_UI_WindowsAndMessaging", "Win32_Foundation"] }
+windows = { version = "0.61", features = [
+ "Win32_UI_WindowsAndMessaging",
+ "Win32_Foundation",
+ # Job Object: assign every spawned test child to a KILL_ON_JOB_CLOSE job so
+ # the OS terminates the whole tree when the test process dies for ANY reason
+ # (including SIGKILL / Ctrl-C) β€” guarantees no orphaned windows or held ports.
+ "Win32_System_JobObjects",
+ # CreateJobObjectW's signature references SECURITY_ATTRIBUTES;
+ # JOBOBJECT_EXTENDED_LIMIT_INFORMATION embeds IO_COUNTERS (Threading).
+ "Win32_Security",
+ "Win32_System_Threading",
+] }
diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/e2e_windows_bg_input_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/e2e_windows_bg_input_test.rs
new file mode 100644
index 0000000000..c0a1a66555
--- /dev/null
+++ b/libs/cua-driver/rust/crates/cua-driver/tests/e2e_windows_bg_input_test.rs
@@ -0,0 +1,558 @@
+//! End-to-end Windows test for the **unified background input interface**:
+//! a caller targets an app and plays click/key actions WITHOUT knowing the
+//! app's internals (Electron/Chromium, Tauri/WebView2, classic Win32) and
+//! WITHOUT the target window ever being raised to the foreground.
+//!
+//! Verified two ways per action:
+//! 1. The tool succeeds in the DEFAULT dispatch mode β€” no
+//! `background_unavailable` error, no `dispatch:"foreground"` needed.
+//! 2. The `focus-monitor-win` sentinel records ZERO foreground losses across
+//! the action == the target window was not z-raised over the user's
+//! window (same oracle as `harness_bg_modality_test`).
+//!
+//! ## Process hygiene (no orphaned windows / held ports)
+//! Every child (target app, sentinel, driver) is spawned into a Windows **Job
+//! Object** created with `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`. The test process
+//! holds the only job handle, so when it exits for ANY reason β€” normal end,
+//! panic, or being force-killed (Ctrl-C) β€” the OS terminates the entire child
+//! tree. A per-fixture `ChildBag` also kills+waits promptly between tests so
+//! windows don't pile up during a run, and every early-return path reaps.
+//!
+//! Targets (trycua test apps, auto-downloaded to %TEMP% if not provided):
+//! - Electron (Chromium content) β€” `trycua/desktop-test-app-electron`.
+//! - Tauri (WebView2 content) β€” `trycua/desktop-test-app`.
+//! - Win32 baseline β€” `notepad.exe`.
+//! Override with `CUA_ELECTRON_EXE` / `CUA_TAURI_EXE`, or drop the exe in
+//! `test-apps/`. Auto-download uses `curl.exe`.
+//!
+//! Every JSON-RPC call is bounded by a hard timeout (a hung driver becomes a
+//! fast, localized failure β€” never an indefinite wall-clock hang).
+//!
+//! All tests are `#[ignore]` (GUI, real desktop session). Run explicitly,
+//! serially (apps bind a fixed HTTP port, so never in parallel):
+//! cargo test -p cua-driver --test e2e_windows_bg_input_test -- --ignored --nocapture --test-threads=1
+
+#![cfg(target_os = "windows")]
+
+use core::ffi::c_void;
+use std::collections::HashSet;
+use std::fs;
+use std::io::{BufRead, BufReader, Write};
+use std::os::windows::io::AsRawHandle;
+use std::path::{Path, PathBuf};
+use std::process::{Child, ChildStdin, Command, Stdio};
+use std::sync::mpsc::{channel, Receiver};
+use std::sync::OnceLock;
+use std::time::{Duration, Instant};
+
+use windows::Win32::Foundation::{CloseHandle, HANDLE};
+use windows::Win32::System::JobObjects::{
+ AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject,
+ JobObjectExtendedLimitInformation, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
+ JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
+};
+use windows::Win32::System::Threading::{OpenProcess, PROCESS_SET_QUOTA, PROCESS_TERMINATE};
+
+const CALL_TIMEOUT: Duration = Duration::from_secs(25);
+
+const ELECTRON_URL: &str = "https://github.com/trycua/desktop-test-app-electron/releases/download/v0.1.0/desktop-test-app-electron.0.1.0.exe";
+const TAURI_URL: &str = "https://github.com/trycua/desktop-test-app/releases/download/v0.2.2/desktop-test-app-windows-x86_64.exe";
+
+// ── Job Object: kill the whole child tree when this test process dies ─────────
+
+/// Process-global job handle (stored as usize so it's `Send`/`Sync` in the
+/// OnceLock). Created lazily with KILL_ON_JOB_CLOSE.
+static JOB: OnceLock<usize> = OnceLock::new();
+
+fn job() -> HANDLE {
+ let raw = *JOB.get_or_init(|| unsafe {
+ let h = CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW");
+ let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
+ info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
+ let _ = SetInformationJobObject(
+ h,
+ JobObjectExtendedLimitInformation,
+ &info as *const _ as *const c_void,
+ std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
+ );
+ h.0 as usize
+ });
+ HANDLE(raw as *mut c_void)
+}
+
+/// Spawn a command and immediately assign it to the kill-on-close job, so it
+/// can never outlive this test process.
+fn spawn_in_job(cmd: &mut Command) -> std::io::Result<Child> {
+ let child = cmd.spawn()?;
+ unsafe {
+ let h = HANDLE(child.as_raw_handle() as *mut c_void);
+ let _ = AssignProcessToJobObject(job(), h);
+ }
+ Ok(child)
+}
+
+/// Assign an already-running pid (e.g. the broker-spawned window process of a
+/// packaged app, which is NOT our direct child) to the kill-on-close job, so
+/// it too dies when this test process exits. Best-effort.
+fn assign_pid_to_job(pid: u32) {
+ unsafe {
+ if let Ok(h) = OpenProcess(PROCESS_SET_QUOTA | PROCESS_TERMINATE, false, pid) {
+ if !h.is_invalid() {
+ let _ = AssignProcessToJobObject(job(), h);
+ let _ = CloseHandle(h);
+ }
+ }
+ }
+}
+
+/// Owns every spawned child AND any externally-launched pids (broker apps);
+/// kills them all on drop β€” prompt cleanup between tests and on early
+/// return / panic unwind. The Job Object is the backstop for hard kills.
+struct ChildBag { children: Vec<Child>, pids: Vec<u32> }
+impl ChildBag {
+ fn new() -> Self { ChildBag { children: Vec::new(), pids: Vec::new() } }
+ fn push(&mut self, c: Child) { self.children.push(c); }
+ /// Track an external pid (and its whole tree) for teardown via taskkill.
+ fn track_pid(&mut self, pid: u32) { self.pids.push(pid); }
+}
+impl Drop for ChildBag {
+ fn drop(&mut self) {
+ // Tree-kill externally-discovered window processes first (packaged /
+ // broker-launched apps whose window pid isn't our spawned child).
+ for pid in &self.pids {
+ let _ = Command::new("taskkill")
+ .args(["/F", "/T", "/PID", &pid.to_string()])
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .status();
+ }
+ for c in &mut self.children {
+ let _ = c.kill();
+ let _ = c.wait();
+ }
+ std::thread::sleep(Duration::from_millis(250));
+ }
+}
+
+/// Best-effort kill of any prior instance of `exe` by basename, so a leftover
+/// from an earlier (e.g. force-killed) run can't hold the app's fixed HTTP port.
+fn kill_prior_by_name(exe: &Path) {
+ if let Some(name) = exe.file_name().and_then(|n| n.to_str()) {
+ let _ = Command::new("taskkill")
+ .args(["/F", "/T", "/IM", name])
+ .stdout(Stdio::null()).stderr(Stdio::null())
+ .status();
+ }
+}
+
+// ── paths / downloads ─────────────────────────────────────────────────────────
+
+fn workspace_root() -> PathBuf {
+ let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");
+ PathBuf::from(manifest).parent().unwrap().parent().unwrap().to_owned()
+}
+fn driver_binary() -> PathBuf { workspace_root().join("target/debug/cua-driver.exe") }
+fn focus_monitor_binary() -> PathBuf { workspace_root().join("target/debug/focus-monitor-win.exe") }
+
+fn curl_download(url: &str, dst: &Path) -> bool {
+ eprintln!("[e2e] downloading {url}\n -> {dst:?}");
+ let ok = Command::new("curl.exe")
+ .args(["-L", "--fail", "--silent", "--show-error", "-o"])
+ .arg(dst).arg(url).status().map(|s| s.success()).unwrap_or(false);
+ ok && fs::metadata(dst).map(|m| m.len() > 4096).unwrap_or(false)
+}
+fn resolve_or_download(env_var: &str, prefix: &str, url: &str, tmp_name: &str) -> Option<PathBuf> {
+ if let Ok(p) = std::env::var(env_var) {
+ let pb = PathBuf::from(p);
+ if pb.exists() { return Some(pb); }
+ }
+ if let Ok(entries) = fs::read_dir(workspace_root().join("test-apps")) {
+ for e in entries.flatten() {
+ let name = e.file_name().to_string_lossy().to_lowercase();
+ if name.starts_with(prefix) && name.ends_with(".exe") && !name.contains("setup") {
+ return Some(e.path());
+ }
+ }
+ }
+ let dst = std::env::temp_dir().join(tmp_name);
+ if dst.exists() && fs::metadata(&dst).map(|m| m.len() > 4096).unwrap_or(false) {
+ return Some(dst);
+ }
+ if curl_download(url, &dst) { Some(dst) } else { None }
+}
+fn electron_exe() -> Option<PathBuf> {
+ resolve_or_download("CUA_ELECTRON_EXE", "desktop-test-app-electron", ELECTRON_URL,
+ "desktop-test-app-electron.0.1.0.exe")
+}
+fn tauri_exe() -> Option<PathBuf> {
+ resolve_or_download("CUA_TAURI_EXE", "desktop-test-app-windows", TAURI_URL,
+ "desktop-test-app-windows-x86_64.exe")
+}
+
+fn loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_losses.txt") }
+fn key_loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_losses.txt") }
+fn focus_pid_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_pid.txt") }
+fn focus_hwnd_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_hwnd.txt") }
+fn read_count(p: &Path) -> u32 {
+ fs::read_to_string(p).ok().and_then(|s| s.trim().parse::<u32>().ok()).unwrap_or(0)
+}
+
+// ── JSON-RPC over the driver's stdio, with per-call timeout ───────────────────
+
+fn send(stdin: &mut ChildStdin, req: serde_json::Value) {
+ let _ = writeln!(stdin, "{}", serde_json::to_string(&req).unwrap());
+ let _ = stdin.flush();
+}
+fn call(stdin: &mut ChildStdin, rx: &Receiver<String>,
+ id: u32, name: &str, args: serde_json::Value) -> serde_json::Value {
+ send(stdin, serde_json::json!({
+ "jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":name,"arguments":args}
+ }));
+ match rx.recv_timeout(CALL_TIMEOUT) {
+ Ok(line) => serde_json::from_str(&line)
+ .unwrap_or_else(|_| serde_json::json!({"error": format!("bad json: {line}")})),
+ Err(_) => serde_json::json!({"error": format!("TIMEOUT (>{}s) on {name}", CALL_TIMEOUT.as_secs())}),
+ }
+}
+fn init(stdin: &mut ChildStdin, rx: &Receiver<String>) {
+ send(stdin, serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));
+ let _ = rx.recv_timeout(CALL_TIMEOUT);
+}
+fn result_text(s: &serde_json::Value) -> String {
+ s["result"]["content"][0]["text"].as_str().unwrap_or("").to_string()
+}
+fn is_error(s: &serde_json::Value) -> bool {
+ s["result"]["isError"].as_bool().unwrap_or(false) || s.get("error").is_some()
+}
+fn find_idx_containing(s: &serde_json::Value, needle: &str) -> Option<u64> {
+ for line in result_text(s).lines() {
+ if !line.to_lowercase().contains(&needle.to_lowercase()) { continue; }
+ let st = line.find('[')? + 1;
+ let en = line[st..].find(']')? + st;
+ if let Ok(n) = line[st..en].trim().parse() { return Some(n); }
+ }
+ None
+}
+
+fn window_ids(stdin: &mut ChildStdin, rx: &Receiver<String>) -> HashSet<u64> {
+ let r = call(stdin, rx, 99, "list_windows", serde_json::json!({}));
+ r["result"]["structuredContent"]["windows"].as_array()
+ .map(|a| a.iter().filter_map(|w| w["window_id"].as_u64()).collect())
+ .unwrap_or_default()
+}
+
+// ── fixture ───────────────────────────────────────────────────────────────────
+
+struct E2eFixture {
+ _bag: ChildBag, // kills target+sentinel+driver on drop
+ stdin: ChildStdin,
+ rx: Receiver<String>,
+ pid: u32,
+ wid: u64,
+}
+
+/// Launch order: driver (no window) β†’ snapshot windows β†’ app β†’ discover the
+/// app's NEW window (works for multi-process apps like Electron whose window
+/// belongs to a child pid) β†’ sentinel (grabs foreground, pushing the app to
+/// the background) β†’ reset counters.
+fn setup(target_exe: &Path, _title_hint: &str) -> Option<E2eFixture> {
+ let driver_bin = driver_binary();
+ let fm_bin = focus_monitor_binary();
+ if !driver_bin.exists() { eprintln!("[e2e] cua-driver.exe not built β€” skipping"); return None; }
+ if !fm_bin.exists() { eprintln!("[e2e] focus-monitor-win.exe not built β€” skipping"); return None; }
+ if target_exe.is_absolute() && !target_exe.exists() {
+ eprintln!("[e2e] target {target_exe:?} missing β€” skipping"); return None;
+ }
+
+ // Defensive: clear any leftover instance holding the app's fixed port.
+ kill_prior_by_name(target_exe);
+ let _ = fs::write(loss_file(), "0");
+ let _ = fs::write(key_loss_file(), "0");
+ let _ = fs::remove_file(focus_pid_file());
+ let _ = fs::remove_file(focus_hwnd_file());
+
+ let mut bag = ChildBag::new();
+
+ // 1. Driver (daemon, no visible window).
+ let mut driver = spawn_in_job(
+ Command::new(&driver_bin).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
+ ).inspect_err(|e| eprintln!("[e2e] driver spawn failed: {e}")).ok()?;
+ let mut stdin = driver.stdin.take().unwrap();
+ let stdout = driver.stdout.take().unwrap();
+ bag.push(driver);
+ let (tx, rx) = channel::<String>();
+ std::thread::spawn(move || {
+ let mut reader = BufReader::new(stdout);
+ let mut line = String::new();
+ loop {
+ line.clear();
+ match reader.read_line(&mut line) {
+ Ok(0) | Err(_) => break,
+ Ok(_) => { if tx.send(line.trim().to_string()).is_err() { break; } }
+ }
+ }
+ });
+ init(&mut stdin, &rx);
+
+ // 2. Snapshot existing windows, then launch the target app.
+ let before = window_ids(&mut stdin, &rx);
+ let app = match spawn_in_job(
+ Command::new(target_exe).stdout(Stdio::null()).stderr(Stdio::null())
+ ) {
+ Ok(c) => c,
+ Err(e) => { eprintln!("[e2e] target spawn failed: {e}"); return None; } // bag drops β†’ kills driver
+ };
+ bag.push(app);
+
+ // 3. Discover the app's NEW window (its pid may be a child of the launched
+ // process β€” Electron/WebView2 are multi-process).
+ let deadline = Instant::now() + Duration::from_secs(15);
+ let mut found: Option<(u32, u64)> = None;
+ while Instant::now() < deadline {
+ let r = call(&mut stdin, &rx, 11, "list_windows", serde_json::json!({}));
+ if let Some(arr) = r["result"]["structuredContent"]["windows"].as_array() {
+ for w in arr {
+ let Some(wid) = w["window_id"].as_u64() else { continue };
+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;
+ let title = w["title"].as_str().unwrap_or("");
+ if !before.contains(&wid) && !title.is_empty() && pid != 0 {
+ found = Some((pid, wid));
+ break;
+ }
+ }
+ }
+ if found.is_some() { break; }
+ std::thread::sleep(Duration::from_millis(500));
+ }
+ let (pid, wid) = match found {
+ Some(p) => p,
+ None => { eprintln!("[e2e] app window never appeared β€” skipping"); return None; } // bag drops
+ };
+ // The window's process is often broker-spawned (packaged apps, Electron),
+ // i.e. NOT our direct child. Job-assign it (hard-kill safety) and track it
+ // for prompt tree-kill on teardown so it can't orphan.
+ assign_pid_to_job(pid);
+ bag.track_pid(pid);
+
+ // 4. Sentinel grabs foreground; app drops to z+1 (the background target).
+ let fm = match spawn_in_job(
+ Command::new(&fm_bin).stdout(Stdio::null()).stderr(Stdio::null())
+ ) {
+ Ok(c) => c,
+ Err(e) => { eprintln!("[e2e] sentinel spawn failed: {e}"); return None; }
+ };
+ bag.push(fm);
+ let sdeadline = Instant::now() + Duration::from_secs(10);
+ loop {
+ let ok = read_count(&focus_pid_file()) != 0
+ && fs::read_to_string(focus_hwnd_file()).ok()
+ .and_then(|s| s.trim().parse::<u64>().ok()).unwrap_or(0) != 0;
+ if ok { break; }
+ if Instant::now() > sdeadline { eprintln!("[e2e] sentinel never published β€” skipping"); return None; }
+ std::thread::sleep(Duration::from_millis(100));
+ }
+ std::thread::sleep(Duration::from_millis(400));
+ let _ = fs::write(loss_file(), "0");
+ let _ = fs::write(key_loss_file(), "0");
+
+ Some(E2eFixture { _bag: bag, stdin, rx, pid, wid })
+}
+
+// ── DOM registration oracle (trycua test apps serve an event log on 6769) ─────
+
+const APP_API: &str = "http://127.0.0.1:6769";
+
+fn http_reset() {
+ let _ = Command::new("curl.exe")
+ .args(["-s", "-m", "3", "-X", "POST", &format!("{APP_API}/reset")])
+ .stdout(Stdio::null()).stderr(Stdio::null()).status();
+}
+/// Returns the `/events` body, or None if the app isn't serving the API.
+fn http_events() -> Option<String> {
+ let out = Command::new("curl.exe")
+ .args(["-s", "-m", "3", &format!("{APP_API}/events")])
+ .output().ok()?;
+ if !out.status.success() { return None; }
+ let body = String::from_utf8_lossy(&out.stdout).into_owned();
+ // Reachable iff it looks like the JSON array the app returns.
+ if body.trim_start().starts_with('[') { Some(body) } else { None }
+}
+/// True if the event log recorded at least one DOM event since the last reset
+/// (i.e. the action actually reached the page). None if the app has no API.
+fn registered_since_reset() -> Option<bool> {
+ http_events().map(|b| b.trim() != "[]" && !b.trim().is_empty())
+}
+
+fn foreground_pid() -> u32 {
+ use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};
+ unsafe {
+ let h = GetForegroundWindow();
+ let mut pid = 0u32;
+ GetWindowThreadProcessId(h, Some(&mut pid));
+ pid
+ }
+}
+
+/// The reliable "no z-raise / no foreground steal" oracle, independent of the
+/// machine's foreground-lock setting: after acting on a BACKGROUND target, the
+/// target's process must never become the foreground window. Polls for ~1.2s
+/// to also catch async self-reactivation (Chromium). `user_pid` is the window
+/// we expect to keep the foreground (the focus-monitor-win sentinel launched
+/// last); reported for diagnostics.
+fn assert_target_stays_background<F: FnOnce()>(label: &str, target_pid: u32, action: F) {
+ let user_pid = read_count(&focus_pid_file());
+ let fg_before = foreground_pid();
+ if fg_before == target_pid {
+ eprintln!("[e2e] WARN {label}: target pid {target_pid} was already foreground before the action (setup couldn't background it)");
+ }
+ action();
+ let deadline = Instant::now() + Duration::from_millis(1200);
+ let mut stole = false;
+ let mut last_fg = fg_before;
+ while Instant::now() < deadline {
+ last_fg = foreground_pid();
+ if last_fg == target_pid { stole = true; break; }
+ std::thread::sleep(Duration::from_millis(80));
+ }
+ assert!(!stole,
+ "{label}: target pid {target_pid} BECAME the foreground window β€” z-raise / foreground steal. \
+ (expected user pid {user_pid}, fg_before={fg_before})");
+ println!("βœ… {label}: target pid {target_pid} stayed background (foreground pid={last_fg}, user={user_pid})");
+}
+
+/// Shared body: default-mode left click into a webview target.
+fn webview_click_case(label: &str, exe: PathBuf) {
+ let mut fx = match setup(&exe, "") { Some(f) => f, None => return };
+ let (pid, wid) = (fx.pid, fx.wid);
+ let snap = call(&mut fx.stdin, &fx.rx, 20, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let elem = find_idx_containing(&snap, "button").or_else(|| find_idx_containing(&snap, "click"));
+
+ http_reset();
+ let mut last = serde_json::Value::Null;
+ assert_target_stays_background(label, pid, || {
+ let args = match elem {
+ Some(idx) => serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx}),
+ None => serde_json::json!({"pid": pid as i64, "window_id": wid, "x": 200, "y": 200}),
+ };
+ last = call(&mut fx.stdin, &fx.rx, 21, "click", args);
+ });
+ assert!(!is_error(&last), "{label}: default-mode click errored: {}", result_text(&last));
+ assert!(!result_text(&last).contains("background_unavailable"),
+ "{label}: click should not need dispatch:foreground, got {:?}", result_text(&last));
+ // Delivery is confirmed by the driver's own βœ… result (UIA Invoke fires the
+ // element's default action / DOM `click`; PostMessage/injection deliver the
+ // button events). The /events pointer-log is reported for info only β€” it
+ // does NOT capture accessibility-driven UIA Invoke (which raises `click`,
+ // not `pointerdown`), so a `[]` there is expected for the UIA path and is
+ // not a failure. The hard invariant of this suite is no-foreground-steal.
+ std::thread::sleep(Duration::from_millis(300));
+ let reg = registered_since_reset().map(|b| b.to_string()).unwrap_or_else(|| "n/a".into());
+ println!("{label}: delivered={:?} pointer-events-logged={reg}", result_text(&last));
+}
+
+// ── tests ─────────────────────────────────────────────────────────────────────
+
+#[test]
+#[ignore]
+fn e2e_electron_background_click_no_z_raise() {
+ let Some(exe) = electron_exe() else { eprintln!("[e2e] no electron app β€” skipping"); return; };
+ webview_click_case("electron click (default dispatch)", exe);
+}
+
+#[test]
+#[ignore]
+fn e2e_tauri_background_click_no_z_raise() {
+ let Some(exe) = tauri_exe() else { eprintln!("[e2e] no tauri app β€” skipping"); return; };
+ webview_click_case("tauri click (default dispatch)", exe);
+}
+
+#[test]
+#[ignore]
+fn e2e_win32_notepad_background_click_no_z_raise() {
+ let mut fx = match setup(Path::new(r"C:\Windows\System32\notepad.exe"), "") { Some(f) => f, None => return };
+ let (pid, wid) = (fx.pid, fx.wid);
+ let mut last = serde_json::Value::Null;
+ assert_target_stays_background("notepad click (default dispatch)", pid, || {
+ last = call(&mut fx.stdin, &fx.rx, 21, "click",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "x": 120, "y": 120}));
+ });
+ assert!(!is_error(&last), "notepad click errored: {}", result_text(&last));
+ println!("notepad click result: {:?}", result_text(&last));
+}
+
+/// Electron right-click (pen-barrel injection): no raise. Pen→right promotion
+/// is app-dependent, so landing is best-effort; the hard invariant is no
+/// z-raise + no background_unavailable.
+#[test]
+#[ignore]
+fn e2e_electron_background_right_click_no_z_raise() {
+ let Some(exe) = electron_exe() else { eprintln!("[e2e] no electron app β€” skipping"); return; };
+ let mut fx = match setup(&exe, "") { Some(f) => f, None => return };
+ let (pid, wid) = (fx.pid, fx.wid);
+ let mut last = serde_json::Value::Null;
+ assert_target_stays_background("electron right-click (default dispatch)", pid, || {
+ last = call(&mut fx.stdin, &fx.rx, 21, "click",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "x": 200, "y": 200, "button": "right"}));
+ });
+ println!("electron right-click result: {:?}", result_text(&last));
+}
+
+/// Electron TEXT typing: focus a field, then type β€” must register the text in
+/// the DOM AND never steal foreground / move the cursor. This is the "type into
+/// any window" half of the mission for plain text (WM_CHAR path, no foreground).
+#[test]
+#[ignore]
+fn e2e_electron_background_type_text_no_z_raise() {
+ let Some(exe) = electron_exe() else { eprintln!("[e2e] no electron app β€” skipping"); return; };
+ let mut fx = match setup(&exe, "") { Some(f) => f, None => return };
+ let (pid, wid) = (fx.pid, fx.wid);
+
+ // Focus a text field if the page exposes one (so the chars have a sink).
+ let snap = call(&mut fx.stdin, &fx.rx, 20, "get_window_state",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));
+ let field = find_idx_containing(&snap, "edit")
+ .or_else(|| find_idx_containing(&snap, "text"))
+ .or_else(|| find_idx_containing(&snap, "input"));
+ if let Some(idx) = field {
+ let _ = call(&mut fx.stdin, &fx.rx, 21, "click",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx}));
+ std::thread::sleep(Duration::from_millis(200));
+ }
+
+ http_reset();
+ let mut last = serde_json::Value::Null;
+ assert_target_stays_background("electron type_text (default dispatch)", pid, || {
+ last = call(&mut fx.stdin, &fx.rx, 22, "type_text",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "text": "cuatest"}));
+ });
+ assert!(!is_error(&last), "type_text errored: {}", result_text(&last));
+ std::thread::sleep(Duration::from_millis(300));
+ if let Some(reg) = registered_since_reset() {
+ // Soft: typing needs a focused sink; if the page had no focusable field
+ // we can't fault the input path. Report either way.
+ println!("electron type_text: registered={reg} result={:?}", result_text(&last));
+ } else {
+ println!("electron type_text result: {:?}", result_text(&last));
+ }
+}
+
+/// Electron key-combo (Ctrl+A): the dropped-PostMessage keyboard path now uses
+/// cloaked focus + SendInput. Hard invariant: no z-raise.
+#[test]
+#[ignore]
+fn e2e_electron_background_keycombo_no_z_raise() {
+ let Some(exe) = electron_exe() else { eprintln!("[e2e] no electron app β€” skipping"); return; };
+ let mut fx = match setup(&exe, "") { Some(f) => f, None => return };
+ let (pid, wid) = (fx.pid, fx.wid);
+ let mut last = serde_json::Value::Null;
+ assert_target_stays_background("electron Ctrl+A (default dispatch)", pid, || {
+ last = call(&mut fx.stdin, &fx.rx, 21, "press_key",
+ serde_json::json!({"pid": pid as i64, "window_id": wid, "key": "a", "modifiers": ["control"]}));
+ });
+ // Capability-first: the combo must be DELIVERED, not refused. (UX is
+ // best-effort: the cloaked focus restores foreground, so the no-steal
+ // assertion above still holds post-action.)
+ assert!(!is_error(&last) && !result_text(&last).contains("background_unavailable"),
+ "Ctrl+A must be delivered (capability over UX), got: {:?}", result_text(&last));
+ println!("electron Ctrl+A delivered: {:?}", result_text(&last));
+}
diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/motion.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/motion.rs
index 59d9586730..eba976ca50 100644
--- a/libs/cua-driver/rust/crates/cursor-overlay/src/motion.rs
+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/motion.rs
@@ -31,6 +31,9 @@ pub struct MotionConfig {
pub min_start_speed: f64,
/// Minimum cursor speed at end of glide (deceleration floor), pts/sec.
pub min_end_speed: f64,
+ /// Minimum turning radius of the Dubins glide path, in points. Smaller =
+ /// tighter curves. Matches the Swift reference default of 80.
+ pub turn_radius: f64,
}
impl Default for MotionConfig {
@@ -48,11 +51,13 @@ impl Default for MotionConfig {
peak_speed: 900.0,
min_start_speed: 300.0,
min_end_speed: 200.0,
+ turn_radius: 80.0,
}
}
}
impl MotionConfig {
+ #[allow(clippy::too_many_arguments)]
pub fn with_overrides(
&self,
start_handle: Option<f64>,
@@ -64,6 +69,7 @@ impl MotionConfig {
dwell_after_click_ms: Option<f64>,
idle_hide_ms: Option<f64>,
press_duration_ms: Option<f64>,
+ turn_radius: Option<f64>,
) -> Self {
fn clamp(v: f64, lo: f64, hi: f64) -> f64 { v.clamp(lo, hi) }
Self {
@@ -79,6 +85,7 @@ impl MotionConfig {
peak_speed: self.peak_speed,
min_start_speed: self.min_start_speed,
min_end_speed: self.min_end_speed,
+ turn_radius: clamp(turn_radius.unwrap_or(self.turn_radius), 1.0, 1000.0),
}
}
}
diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs
index 82fb78cd78..3d3579a054 100644
--- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs
+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs
@@ -133,7 +133,8 @@ impl RenderStateCore {
let mut fire_arrival = false;
if let Some(ref p) = self.path {
- let path_frac = (self.dist / p.length.max(1.0)).clamp(0.0, 1.0);
+ let path_len = p.length.max(1.0);
+ let path_frac = (self.dist / path_len).clamp(0.0, 1.0);
let profile =
16.0 * path_frac * path_frac * (1.0 - path_frac) * (1.0 - path_frac);
let floor = if path_frac < 0.5 {
@@ -141,19 +142,37 @@ impl RenderStateCore {
} else {
self.motion.min_end_speed
};
- let speed = (floor + (self.motion.peak_speed - floor) * profile).max(floor);
+ let speed_based = (floor + (self.motion.peak_speed - floor) * profile).max(floor);
+ // Fixed-duration override: when `glide_duration_ms > 0` the move
+ // takes exactly that long regardless of distance, so an orchestrator
+ // can lock glides to a known cadence. `0` (the default) keeps the
+ // speed-based timing untouched. Shared verbatim with the macOS
+ // reference path (`tick_swift_constants`) β€” no platform drift.
+ let speed = if self.motion.glide_duration_ms > 0.0 {
+ path_len / (self.motion.glide_duration_ms / 1000.0)
+ } else {
+ speed_based
+ };
self.dist += speed * dt;
- let path_len = p.length.max(1.0);
if self.dist >= path_len {
let end = p.sample(path_len);
let end_heading = p.end_visual_heading;
let vh = end.heading;
+ // In fixed-duration mode the constant speed can be large; base
+ // the settle impulse on the normal end-floor so the landing
+ // stays as crisp as a speed-based glide instead of overshooting
+ // proportionally to a short duration.
+ let impulse = if self.motion.glide_duration_ms > 0.0 {
+ self.motion.min_end_speed
+ } else {
+ speed
+ };
self.spring = Some(Spring {
ox: 0.0,
oy: 0.0,
- vx: speed * 0.5 * vh.cos(),
- vy: speed * 0.5 * vh.sin(),
+ vx: impulse * 0.5 * vh.cos(),
+ vy: impulse * 0.5 * vh.sin(),
});
self.spring_tgt = Some((end.x, end.y, end_heading));
self.pos = (end.x, end.y);
@@ -164,9 +183,11 @@ impl RenderStateCore {
} else {
let s: PathState = p.sample(self.dist);
self.pos = (s.x, s.y);
- let desired = s.heading + std::f64::consts::PI;
- let max_step = 14.0 * dt;
- self.heading = crate::util::rotate_toward(self.heading, desired, max_step);
+ // Point the arrow exactly along the path tangent (the renderer
+ // adds Ο€, so we store tangent+Ο€). Assigned directly rather than
+ // rate-limited toward it, so the tip actually tracks the
+ // trajectory instead of lagging behind on fast/short glides.
+ self.heading = s.heading + std::f64::consts::PI;
}
} else if let Some(mut s) = self.spring {
if let Some((tx, ty, th)) = self.spring_tgt {
@@ -228,7 +249,17 @@ impl RenderStateCore {
// Smootherstep speed profile (normalised: peak = 1.0).
let profile = (30.0 * u * u * (1.0 - u) * (1.0 - u)) / 1.875;
let floor_speed = if u < 0.5 { MIN_START_SPEED } else { MIN_END_SPEED };
- let current_speed = floor_speed + (PEAK_SPEED - floor_speed) * profile;
+ let speed_based = floor_speed + (PEAK_SPEED - floor_speed) * profile;
+ // Fixed-duration override: when `glide_duration_ms > 0` the move
+ // takes exactly that long regardless of distance, so an orchestrator
+ // can lock glides to a known cadence. `0` (the default) keeps the
+ // speed-based timing untouched. Shared verbatim with the
+ // Windows/Linux path (`tick_motion`) β€” no platform drift.
+ let current_speed = if self.motion.glide_duration_ms > 0.0 {
+ path_len / (self.motion.glide_duration_ms / 1000.0)
+ } else {
+ speed_based
+ };
self.dist += current_speed * dt;
if self.dist >= path_len {
@@ -236,11 +267,20 @@ impl RenderStateCore {
let end = p.sample(path_len);
let end_heading = p.end_visual_heading;
let vh = end.heading;
+ // In fixed-duration mode the constant speed can be large; base
+ // the settle impulse on the normal end-floor so the landing
+ // stays as crisp as a speed-based glide instead of overshooting
+ // proportionally to a short duration.
+ let impulse = if self.motion.glide_duration_ms > 0.0 {
+ MIN_END_SPEED
+ } else {
+ current_speed
+ };
self.spring = Some(Spring {
ox: 0.0,
oy: 0.0,
- vx: current_speed * SPRING_OVERSHOOT * vh.cos(),
- vy: current_speed * SPRING_OVERSHOOT * vh.sin(),
+ vx: impulse * SPRING_OVERSHOOT * vh.cos(),
+ vy: impulse * SPRING_OVERSHOOT * vh.sin(),
});
self.spring_tgt = Some((end.x, end.y, end_heading));
self.pos = (end.x, end.y);
@@ -251,10 +291,10 @@ impl RenderStateCore {
} else {
let s: PathState = p.sample(self.dist);
self.pos = (s.x, s.y);
- // Smooth heading rotation toward motion heading.
- let desired = s.heading + std::f64::consts::PI;
- let max_step = 14.0 * dt;
- self.heading = crate::util::rotate_toward(self.heading, desired, max_step);
+ // Point the arrow exactly along the path tangent (renderer adds
+ // Ο€, so store tangent+Ο€). Direct assignment, not rate-limited, so
+ // the tip tracks the trajectory instead of lagging on fast moves.
+ self.heading = s.heading + std::f64::consts::PI;
}
} else if let Some(mut s) = self.spring {
if let Some((tx, ty, th)) = self.spring_tgt {
@@ -347,7 +387,7 @@ impl RenderStateCore {
// tx = clickPoint.x + cos(endAngle) * clickOffset
// ty = clickPoint.y + sin(endAngle) * clickOffset
const CLICK_OFFSET: f64 = 16.0;
- const TURN_RADIUS: f64 = 80.0;
+ let turn_radius = self.motion.turn_radius;
let tx = x + end_heading_radians.cos() * CLICK_OFFSET;
let ty = y + end_heading_radians.sin() * CLICK_OFFSET;
@@ -367,7 +407,7 @@ impl RenderStateCore {
ty,
th1,
end_heading_radians,
- TURN_RADIUS,
+ turn_radius,
);
self.path = Some(plan);
self.dist = 0.0;
@@ -759,3 +799,52 @@ pub fn draw_default_arrow(
None,
);
}
+
+#[cfg(test)]
+mod glide_duration_tests {
+ use super::*;
+ use crate::{CursorConfig, PathPlanner};
+
+ /// Run a glide of `dist_pts` to completion and return how many seconds it
+ /// took. `tick` selects the platform path: `false` = `tick_motion`
+ /// (Windows/Linux), `true` = `tick_swift_constants` (macOS reference).
+ fn arrival_secs(glide_ms: f64, dist_pts: f64, swift: bool) -> f64 {
+ let mut core = RenderStateCore::new(CursorConfig::default());
+ core.motion.glide_duration_ms = glide_ms;
+ core.motion.idle_hide_ms = 0.0;
+ core.pos = (0.0, 0.0);
+ // Aligned headings β†’ an effectively straight path of length ~dist_pts.
+ core.path = Some(PathPlanner::plan(0.0, 0.0, 0.0, dist_pts, 0.0, 0.0, 0.0, 80.0));
+ core.dist = 0.0;
+ let dt = 1.0 / 240.0;
+ let mut t = 0.0;
+ for _ in 0..200_000 {
+ let arrived = if swift { core.tick_swift_constants(dt) } else { core.tick_motion(dt) };
+ t += dt;
+ if arrived { break; }
+ }
+ t
+ }
+
+ #[test]
+ fn fixed_duration_is_distance_independent_on_both_paths() {
+ for swift in [false, true] {
+ let short = arrival_secs(300.0, 120.0, swift);
+ let long = arrival_secs(300.0, 1400.0, swift);
+ // Both land in ~300ms regardless of distance (within a few ticks).
+ assert!((short - 0.3).abs() < 0.05, "swift={swift} short={short}");
+ assert!((long - 0.3).abs() < 0.05, "swift={swift} long={long}");
+ }
+ }
+
+ #[test]
+ fn zero_keeps_speed_based_timing() {
+ // glide_duration_ms == 0 (the default) β†’ longer paths take longer, on
+ // both platform paths, exactly as before this field was implemented.
+ for swift in [false, true] {
+ let short = arrival_secs(0.0, 120.0, swift);
+ let long = arrival_secs(0.0, 1400.0, swift);
+ assert!(long > short + 0.2, "swift={swift} short={short} long={long}");
+ }
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs
index 58c9e99f8d..c927f0144a 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs
@@ -123,7 +123,7 @@ fn motion_def() -> &'static ToolDef {
- arc_size: perpendicular deflection as fraction of path length [0,1]. Default 0.25\n\
- arc_flow: asymmetry [-1,1]; positive bulges toward destination. Default 0.0\n\
- spring: settle damping [0.3,1.0]; 1.0=no overshoot. Default 0.72\n\
- - glide_duration_ms: flight duration per move [50,5000]. Default 160\n\
+ - glide_duration_ms: fixed flight duration per move [50,5000]; omit for speed-based (the default)\n\
- dwell_after_click_ms: pause after click ripple [0,5000]. Default 80\n\
- idle_hide_ms: auto-hide delay [0,60000]; 0=never. Default 20000".into(),
input_schema: serde_json::json!({
@@ -159,7 +159,7 @@ fn motion_def() -> &'static ToolDef {
"type": "number",
"minimum": 50,
"maximum": 5000,
- "description": "Flight duration per move in ms. Default 160."
+ "description": "Fixed flight duration per move in ms; omit for speed-based timing (the default)."
},
"dwell_after_click_ms": {
"type": "number",
@@ -172,6 +172,12 @@ fn motion_def() -> &'static ToolDef {
"minimum": 0,
"maximum": 60000,
"description": "Auto-hide delay in ms. 0 = never hide. Default 20000."
+ },
+ "turn_radius": {
+ "type": "number",
+ "minimum": 1,
+ "maximum": 1000,
+ "description": "Minimum turning radius of the glide path in points; smaller = tighter curves. Default 80."
}
},
"additionalProperties": false
@@ -226,7 +232,8 @@ impl Tool for SetAgentCursorMotionTool {
|| args.get("spring").is_some()
|| args.get("glide_duration_ms").is_some()
|| args.get("dwell_after_click_ms").is_some()
- || args.get("idle_hide_ms").is_some();
+ || args.get("idle_hide_ms").is_some()
+ || args.get("turn_radius").is_some();
if motion_changed {
// Read this cursor's current motion from the overlay, apply
@@ -242,6 +249,7 @@ impl Tool for SetAgentCursorMotionTool {
num(args.get("dwell_after_click_ms")),
num(args.get("idle_hide_ms")),
None, // press_duration_ms not exposed
+ num(args.get("turn_radius")),
);
crate::cursor::overlay::send_command(
cursor_id.clone(),
diff --git a/libs/cua-driver/rust/crates/platform-windows/Cargo.toml b/libs/cua-driver/rust/crates/platform-windows/Cargo.toml
index c40c8eccef..bd64e08e43 100644
--- a/libs/cua-driver/rust/crates/platform-windows/Cargo.toml
+++ b/libs/cua-driver/rust/crates/platform-windows/Cargo.toml
@@ -32,6 +32,12 @@ windows = { version = "0.58", features = [
"Win32_Graphics_Gdi",
"Win32_Graphics_Dwm",
"Win32_UI_Input_KeyboardAndMouse",
+ # Pointer/touch injection (Track A background-actuation probe):
+ # InitializeTouchInjection / InjectTouchInput / InjectSyntheticPointerInput
+ # live in Win32_UI_Input_Pointer; POINTER_TYPE_INFO / HSYNTHETICPOINTERDEVICE
+ # / CreateSyntheticPointerDevice live in Win32_UI_Controls.
+ "Win32_UI_Input_Pointer",
+ "Win32_UI_Controls",
"Win32_UI_HiDpi",
"Win32_System_LibraryLoader",
"Win32_Graphics_OpenGL",
diff --git a/libs/cua-driver/rust/crates/platform-windows/examples/zdrop_probe.rs b/libs/cua-driver/rust/crates/platform-windows/examples/zdrop_probe.rs
new file mode 100644
index 0000000000..8fbdef461c
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-windows/examples/zdrop_probe.rs
@@ -0,0 +1,314 @@
+//! Z-drop harness + Track A background-input probe.
+//!
+//! Measures the foreground "flash": when actuating input into a *background*
+//! window, does the target window rise above the user's foreground window?
+//!
+//! The harness mirrors the `flash-repro` methodology referenced in
+//! `uia/fg_bypass.rs`: capture the user's foreground window as `baseline`, then
+//! poll the z-order at high frequency while an actuation runs on another thread.
+//! A "z-drop" is any sample where `target` sits above `baseline` (or `baseline`
+//! is no longer the foreground). 0 z-drops == no visible flash.
+//!
+//! Two actuators are compared against the SAME background target:
+//! - control: SendInput with a brief SetForegroundWindow swap (the existing
+//! flash path in `input/mouse.rs::send_click_synthesized`).
+//! - inject : Track A β€” pointer/touch injection (InitializeTouchInjection +
+//! InjectTouchInput), which RE showed has no foreground precondition
+//! (NtUserInjectMouseInput gates only on per-process injection-enable, not
+//! GetForegroundWindow β€” see docs/windows-background-input-re-plan.rs Β§4.4).
+//!
+//! Usage (run from an interactive desktop session, with a Chrome/WPF/etc window
+//! open in the BACKGROUND and a different window focused in front):
+//! cargo run -p platform-windows --example zdrop_probe -- [mode] [pid]
+//! mode = both (default) | control | inject | list
+//!
+//! It prints the candidate windows, the baseline + target, a 2s countdown
+//! (position your windows), then the z-drop numbers per actuator.
+//!
+//! WARNING: this performs a real left-click/tap at the target window's center.
+//! Pick a target where a center click is harmless, or pass an explicit pid.
+
+#[cfg(target_os = "windows")]
+fn main() {
+ probe::run();
+}
+
+#[cfg(not(target_os = "windows"))]
+fn main() {
+ eprintln!("zdrop_probe is Windows-only");
+}
+
+#[cfg(target_os = "windows")]
+mod probe {
+ use std::thread;
+ use std::time::{Duration, Instant};
+
+ use windows::Win32::Foundation::{BOOL, HANDLE, HWND, LPARAM, POINT, RECT, TRUE};
+ use windows::Win32::UI::HiDpi::{
+ SetProcessDpiAwarenessContext, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2,
+ };
+ use windows::Win32::UI::Input::Pointer::{
+ InitializeTouchInjection, InjectTouchInput, POINTER_FLAG_DOWN, POINTER_FLAG_INCONTACT,
+ POINTER_FLAG_INRANGE, POINTER_FLAG_UP, POINTER_INFO, POINTER_TOUCH_INFO,
+ TOUCH_FEEDBACK_DEFAULT,
+ };
+ use windows::Win32::UI::Input::KeyboardAndMouse::{
+ SendInput, INPUT, INPUT_0, INPUT_MOUSE, MOUSEINPUT, MOUSEEVENTF_LEFTDOWN,
+ MOUSEEVENTF_LEFTUP,
+ };
+ use windows::Win32::UI::WindowsAndMessaging::{
+ EnumWindows, GetClassNameW, GetCursorPos, GetForegroundWindow, GetWindow, GetWindowRect,
+ GetWindowTextW, GetWindowThreadProcessId, GetTopWindow, IsWindowVisible, SetCursorPos,
+ SetForegroundWindow, GW_HWNDNEXT, PT_TOUCH,
+ };
+
+ // ---- z-order helpers ----------------------------------------------------
+
+ /// Walk the top-level z-order from the top; return true if `a` is above `b`.
+ /// None if either isn't found in the walk.
+ unsafe fn is_above(a: HWND, b: HWND) -> Option<bool> {
+ let mut h = GetTopWindow(None).ok()?;
+ loop {
+ if h == a {
+ return Some(true);
+ }
+ if h == b {
+ return Some(false);
+ }
+ match GetWindow(h, GW_HWNDNEXT) {
+ Ok(n) if !n.0.is_null() => h = n,
+ _ => return None,
+ }
+ }
+ }
+
+ struct PollResult {
+ samples: u64,
+ target_above: u64,
+ fg_not_baseline: u64,
+ }
+
+ /// Sample the z-order for `dur`, counting flashes of `target` over `baseline`.
+ fn poll(baseline: usize, target: usize, dur: Duration) -> PollResult {
+ let baseline = HWND(baseline as *mut _);
+ let target = HWND(target as *mut _);
+ let deadline = Instant::now() + dur;
+ let mut r = PollResult { samples: 0, target_above: 0, fg_not_baseline: 0 };
+ while Instant::now() < deadline {
+ unsafe {
+ r.samples += 1;
+ if GetForegroundWindow() != baseline {
+ r.fg_not_baseline += 1;
+ }
+ if is_above(target, baseline) == Some(true) {
+ r.target_above += 1;
+ }
+ }
+ thread::sleep(Duration::from_millis(4));
+ }
+ r
+ }
+
+ // ---- actuators ----------------------------------------------------------
+
+ /// Control: the existing flash path β€” SetForegroundWindow swap + SendInput.
+ fn actuate_sendinput_swap(target: usize, x: i32, y: i32) {
+ let target = HWND(target as *mut _);
+ unsafe {
+ let prev_fg = GetForegroundWindow();
+ let mut prev_cursor = POINT::default();
+ let _ = GetCursorPos(&mut prev_cursor);
+ let _ = SetForegroundWindow(target);
+ thread::sleep(Duration::from_millis(8));
+ let _ = SetCursorPos(x, y);
+ let down = INPUT {
+ r#type: INPUT_MOUSE,
+ Anonymous: INPUT_0 {
+ mi: MOUSEINPUT { dwFlags: MOUSEEVENTF_LEFTDOWN, ..Default::default() },
+ },
+ };
+ let up = INPUT {
+ r#type: INPUT_MOUSE,
+ Anonymous: INPUT_0 {
+ mi: MOUSEINPUT { dwFlags: MOUSEEVENTF_LEFTUP, ..Default::default() },
+ },
+ };
+ let events = [down, up];
+ SendInput(&events, std::mem::size_of::<INPUT>() as i32);
+ thread::sleep(Duration::from_millis(40));
+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);
+ if !prev_fg.0.is_null() && prev_fg != target {
+ let _ = SetForegroundWindow(prev_fg);
+ }
+ }
+ }
+
+ /// Track A: coordinate-routed touch injection. No SetForegroundWindow.
+ /// `hwndTarget` is left NULL so the system hit-tests by screen coordinate.
+ fn actuate_touch_inject(x: i32, y: i32) -> Result<(), String> {
+ unsafe {
+ // Per-process injection enable (idempotent; ignore "already init").
+ let _ = InitializeTouchInjection(1, TOUCH_FEEDBACK_DEFAULT);
+ let mk = |flags| POINTER_TOUCH_INFO {
+ pointerInfo: POINTER_INFO {
+ pointerType: PT_TOUCH,
+ pointerId: 0,
+ pointerFlags: flags,
+ sourceDevice: HANDLE::default(),
+ hwndTarget: HWND::default(),
+ ptPixelLocation: POINT { x, y },
+ ..Default::default()
+ },
+ touchFlags: 0,
+ touchMask: 0,
+ rcContact: RECT { left: x - 2, top: y - 2, right: x + 2, bottom: y + 2 },
+ rcContactRaw: RECT { left: x - 2, top: y - 2, right: x + 2, bottom: y + 2 },
+ orientation: 0,
+ pressure: 512,
+ };
+ let down = mk(POINTER_FLAG_DOWN | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT);
+ InjectTouchInput(&[down]).map_err(|e| format!("InjectTouchInput(down): {e}"))?;
+ thread::sleep(Duration::from_millis(30));
+ let up = mk(POINTER_FLAG_UP);
+ InjectTouchInput(&[up]).map_err(|e| format!("InjectTouchInput(up): {e}"))?;
+ Ok(())
+ }
+ }
+
+ // ---- window discovery ----------------------------------------------------
+
+ unsafe extern "system" fn collect(hwnd: HWND, lparam: LPARAM) -> BOOL {
+ let v = &mut *(lparam.0 as *mut Vec<HWND>);
+ if IsWindowVisible(hwnd).as_bool() {
+ v.push(hwnd);
+ }
+ TRUE
+ }
+
+ fn class_of(h: HWND) -> String {
+ let mut buf = [0u16; 128];
+ let n = unsafe { GetClassNameW(h, &mut buf) };
+ String::from_utf16_lossy(&buf[..n.max(0) as usize])
+ }
+ fn title_of(h: HWND) -> String {
+ let mut buf = [0u16; 256];
+ let n = unsafe { GetWindowTextW(h, &mut buf) };
+ String::from_utf16_lossy(&buf[..n.max(0) as usize])
+ }
+ fn pid_of(h: HWND) -> u32 {
+ let mut pid = 0u32;
+ unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) };
+ pid
+ }
+
+ fn center(h: HWND) -> Option<(i32, i32)> {
+ let mut r = RECT::default();
+ unsafe { GetWindowRect(h, &mut r).ok()? };
+ if r.right <= r.left || r.bottom <= r.top {
+ return None;
+ }
+ Some(((r.left + r.right) / 2, (r.top + r.bottom) / 2))
+ }
+
+ fn run_trial(name: &str, baseline: HWND, target: HWND, x: i32, y: i32, actuate: impl FnOnce()) {
+ let b = baseline.0 as usize;
+ let t = target.0 as usize;
+ let poller = thread::spawn(move || poll(b, t, Duration::from_millis(1200)));
+ thread::sleep(Duration::from_millis(120)); // let poller establish "baseline on top"
+ let _ = (x, y);
+ actuate();
+ let r = poller.join().unwrap();
+ let pct = |n: u64| if r.samples == 0 { 0.0 } else { 100.0 * n as f64 / r.samples as f64 };
+ println!(
+ " [{name:8}] samples={:4} target_above_baseline={:4} ({:5.1}%) fg!=baseline={:4} ({:5.1}%)",
+ r.samples, r.target_above, pct(r.target_above), r.fg_not_baseline, pct(r.fg_not_baseline)
+ );
+ }
+
+ pub fn run() {
+ unsafe {
+ let _ = SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2);
+ }
+ let args: Vec<String> = std::env::args().skip(1).collect();
+ let mode = args.get(0).map(|s| s.as_str()).unwrap_or("both");
+ let forced_pid: Option<u32> = args.get(1).and_then(|s| s.parse().ok());
+
+ let mut wins: Vec<HWND> = Vec::new();
+ unsafe {
+ let _ = EnumWindows(Some(collect), LPARAM(&mut wins as *mut _ as isize));
+ }
+ let fg = unsafe { GetForegroundWindow() };
+
+ // Candidate = visible, titled, not the current foreground, not shell.
+ let preferred = ["chrome", "msedge", "firefox", "notepad", "wpf", "soffice"];
+ let candidates: Vec<HWND> = wins
+ .iter()
+ .copied()
+ .filter(|&h| h != fg && !title_of(h).is_empty())
+ .filter(|&h| {
+ let c = class_of(h).to_lowercase();
+ !c.contains("progman") && !c.contains("workerw") && !c.contains("shell_traywnd")
+ })
+ .collect();
+
+ if mode == "list" || candidates.is_empty() {
+ println!("Foreground (baseline): pid={} class={:?} title={:?}", pid_of(fg), class_of(fg), title_of(fg));
+ println!("Candidate background windows:");
+ for h in &candidates {
+ println!(" pid={:6} class={:24} title={:?}", pid_of(*h), class_of(*h), title_of(*h));
+ }
+ if candidates.is_empty() {
+ eprintln!("\nNo background candidate windows. Open one and focus a different window.");
+ }
+ if mode == "list" {
+ return;
+ }
+ }
+
+ let target = match forced_pid {
+ Some(pid) => candidates.iter().copied().find(|&h| pid_of(h) == pid),
+ None => candidates
+ .iter()
+ .copied()
+ .find(|&h| {
+ let t = title_of(h).to_lowercase();
+ let c = class_of(h).to_lowercase();
+ preferred.iter().any(|p| t.contains(p) || c.contains(p))
+ })
+ .or_else(|| candidates.first().copied()),
+ };
+ let Some(target) = target else {
+ eprintln!("No target window selected.");
+ return;
+ };
+
+ let Some((x, y)) = center(target) else {
+ eprintln!("Target has no usable rect.");
+ return;
+ };
+
+ println!("baseline (user fg): pid={} title={:?}", pid_of(fg), title_of(fg));
+ println!(
+ "target (background): pid={} class={:?} title={:?} click@({x},{y})",
+ pid_of(target), class_of(target), title_of(target)
+ );
+ println!("Position your windows now β€” actuating in 2s. (target should stay BEHIND baseline)");
+ thread::sleep(Duration::from_secs(2));
+
+ if mode == "both" || mode == "control" {
+ run_trial("control", fg, target, x, y, || actuate_sendinput_swap(target.0 as usize, x, y));
+ thread::sleep(Duration::from_millis(400));
+ }
+ if mode == "both" || mode == "inject" {
+ // re-read baseline: control may have left fg elsewhere; refocus check
+ let base2 = unsafe { GetForegroundWindow() };
+ run_trial("inject", base2, target, x, y, || {
+ if let Err(e) = actuate_touch_inject(x, y) {
+ eprintln!(" touch-inject error: {e}");
+ }
+ });
+ }
+ println!("\nInterpretation: control should show a high target_above%/fg!=baseline% (the flash).");
+ println!("Track A (inject) showing ~0% target_above == background input with no visible raise.");
+ }
+}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/dispatch.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/dispatch.rs
index ece145b905..7f4410357d 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/dispatch.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/dispatch.rs
@@ -92,15 +92,17 @@ pub fn dispatch_schema() -> Value {
"enum": ["background", "foreground", "auto"],
"default": "background",
"description":
- "Dispatch mode. 'background' (default) refuses to swap \
- foreground; returns a structured background_unavailable error \
- if PostMessage would be silently dropped for this event kind \
- on this target. 'foreground' explicitly accepts a brief \
- SetForegroundWindow swap (SendInput path) β€” required to drive \
- Chromium-content or GTK-button widget targets reliably. \
- 'auto' uses cua-driver's internal heuristics (silent fallback \
- to SendInput on known-problematic targets); opt-in for \
- callers that prefer the historical behavior."
+ "Dispatch mode. 'background' (default) never swaps foreground: it \
+ routes through UIA Invoke / PostMessage, and for targets that \
+ silently drop posted clicks (Chromium/Electron content, GTK \
+ buttons) it transparently falls back to coordinate-based pointer \
+ injection β€” so a caller can just target the app and click without \
+ knowing its internals, and the window is never raised. (A \
+ background_unavailable error only surfaces for inputs injection \
+ can't express, e.g. a right/middle click on such a target.) \
+ 'foreground' explicitly accepts a brief SetForegroundWindow swap \
+ (SendInput path). 'auto' uses cua-driver's historical heuristics \
+ (silent SendInput fallback on known-problematic targets)."
})
}
@@ -118,6 +120,17 @@ pub fn would_be_silently_dropped(hwnd: u64, kind: EventKind) -> bool {
// Chromium's IME path, which DOES consume Win32 messages.
return matches!(kind, MouseClick | MouseMove | MouseScroll | KeyCombo);
}
+ if is_wpf_target_window(hwnd) {
+ // WPF ignores PostMessage mouse (its input manager drops mouse messages
+ // unless the live system cursor is over the window β€” verified: posted
+ // WM_MOUSE* raise no WPF events). It must be driven by coordinate-routed
+ // system-queue input. We use a PERSISTENT synthetic touch digitizer
+ // (see inject::TOUCH_DEV): WPF's stylus stack binds to the standing
+ // device and consumes the contact as touch/stylus β€” promoting to mouse
+ // internally, with NO OS cursor movement. WM_CHAR keystrokes still work,
+ // so flag only the pointer-class events.
+ return matches!(kind, MouseClick | MouseMove | MouseScroll);
+ }
if is_gtk_target_window(hwnd) {
// Conservative flag for GTK: button widgets ignore PostMessage
// clicks, drawing-area widgets accept them. We cannot distinguish
@@ -165,6 +178,15 @@ pub fn is_vcl_target_window(hwnd: u64) -> bool {
class.starts_with("SAL")
}
+/// Detect WPF top-level windows. WPF hosts its visual tree in an HWND whose
+/// class is `HwndWrapper[<module>;;<guid>]`. WPF TextBoxes consume only real
+/// keyboard input routed through WPF's input manager, so a posted `WM_CHAR`
+/// (the `post_type_text` path) is silently dropped β€” type_text must instead
+/// deliver genuine SendInput keystrokes (see `inject_text_cloaked`).
+pub fn is_wpf_target_window(hwnd: u64) -> bool {
+ read_class_name(hwnd).starts_with("HwndWrapper")
+}
+
/// Detect GTK/GDK top-level windows.
///
/// GTK 3 on Windows uses class `gdkWindowToplevel`; GTK 4 uses
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs
new file mode 100644
index 0000000000..fe562a236e
--- /dev/null
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs
@@ -0,0 +1,792 @@
+//! Universal background mouse actuator: coordinate-routed pointer/touch
+//! injection that delivers a click to whatever window sits under a screen
+//! point **without** `SetForegroundWindow` and **without** moving the user's
+//! mouse cursor.
+//!
+//! Why this exists: the PostMessage path (`mouse::post_click`) is invisible and
+//! never raises, but Chromium/Electron/GTK/WPF content silently ignore posted
+//! synthetic `WM_*BUTTON` messages β€” their input arrives through the *system
+//! input queue*, not the per-window message queue. The historical fallback for
+//! those was `send_click_synthesized` (SendInput + a `SetForegroundWindow`
+//! swap), which is exactly the visible "flash" we want to eliminate.
+//!
+//! Touch injection routes by coordinate through the system input queue (so
+//! Chromium et al. accept it; the OS promotes it to `WM_*BUTTON` for legacy
+//! Win32 windows that don't consume `WM_POINTER`), and β€” per the RE in
+//! `docs/windows-background-input-re-plan.md` Β§4.4 β€” the kernel injection path
+//! (`NtUserInjectMouseInput`/`NtUserInjectTouchInput`) gates only on a
+//! per-process injection-enable, NOT on the target being foreground. The one
+//! residual is that a tap on an *inactive* top-level window can still trigger
+//! click-activation; we contain that with [`ZorderGuard`], which DWM-cloaks the
+//! (background) target for the duration of the tap so any transient raise is
+//! invisible, then restores the user's foreground window and uncloaks.
+//!
+//! Scope: left-button taps (single/double/triple). Right/middle have no clean
+//! touch mapping; callers fall back to their existing routing for those.
+
+use anyhow::{bail, Result};
+use core::ffi::c_void;
+use std::sync::{Mutex, MutexGuard, TryLockError};
+use std::thread::sleep;
+use std::time::{Duration, Instant};
+
+/// Serializes the cloaked-foreground SendInput operations (`inject_key_cloaked`,
+/// `inject_text_cloaked`). Concurrent sessions must not interleave foreground
+/// swaps + SendInput on the single shared system input queue, or keystrokes get
+/// garbled and foreground restores race. Acquired with a hard 1s ceiling so a
+/// stuck holder can never deadlock the others β€” after 1s, callers proceed
+/// unserialized (degraded, but never hung).
+static FG_SERIAL: Mutex<()> = Mutex::new(());
+
+fn fg_serialize() -> Option<MutexGuard<'static, ()>> {
+ let deadline = Instant::now() + Duration::from_secs(1);
+ loop {
+ match FG_SERIAL.try_lock() {
+ Ok(g) => return Some(g),
+ Err(TryLockError::Poisoned(p)) => return Some(p.into_inner()),
+ Err(TryLockError::WouldBlock) => {
+ if Instant::now() >= deadline {
+ return None; // auto-expire: proceed without the lock
+ }
+ sleep(Duration::from_millis(20));
+ }
+ }
+ }
+}
+
+use windows::Win32::Foundation::{BOOL, FALSE, HANDLE, HWND, POINT, RECT, TRUE};
+use windows::Win32::Graphics::Dwm::{DwmSetWindowAttribute, DWMWA_CLOAK};
+use windows::Win32::UI::Controls::{
+ CreateSyntheticPointerDevice, DestroySyntheticPointerDevice, HSYNTHETICPOINTERDEVICE,
+ POINTER_FEEDBACK_DEFAULT, POINTER_TYPE_INFO, POINTER_TYPE_INFO_0,
+};
+use windows::Win32::UI::Input::Pointer::{
+ InjectSyntheticPointerInput, POINTER_FLAG_DOWN, POINTER_FLAG_INCONTACT, POINTER_FLAG_INRANGE,
+ POINTER_FLAG_UP, POINTER_FLAG_UPDATE, POINTER_INFO, POINTER_PEN_INFO, POINTER_TOUCH_INFO,
+};
+use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId};
+use windows::Win32::UI::Input::KeyboardAndMouse::{
+ SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYEVENTF_KEYUP, KEYEVENTF_UNICODE,
+ VIRTUAL_KEY,
+};
+use windows::Win32::UI::WindowsAndMessaging::{
+ GetAncestor, GetCursorPos, GetForegroundWindow, GetWindowLongPtrW, GetWindowThreadProcessId,
+ IsWindow, SetCursorPos, SetForegroundWindow, SetWindowLongPtrW, SetWindowPos,
+ SystemParametersInfoW, GA_ROOT, GWL_EXSTYLE, HWND_NOTOPMOST, HWND_TOP, HWND_TOPMOST, PT_PEN,
+ PT_TOUCH, SPI_GETFOREGROUNDLOCKTIMEOUT, SPI_SETFOREGROUNDLOCKTIMEOUT, SWP_NOACTIVATE, SWP_NOMOVE,
+ SWP_NOSIZE, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS, WS_EX_NOACTIVATE,
+};
+
+/// Bring `target` to the foreground using the AttachThreadInput trick, which
+/// inherits the current foreground thread's FG-lock token so the swap is
+/// honored even on a foreground-locked session without UIAccess (mirrors the
+/// `bring_to_front` tool). Single attach, no retry loop β€” bounded. Returns
+/// whether `target` actually became foreground.
+unsafe fn force_foreground_attached(target: HWND) -> bool {
+ let cur = GetForegroundWindow();
+ if cur == target {
+ return true;
+ }
+ let my_tid = GetCurrentThreadId();
+ let mut pid = 0u32;
+ let cur_tid = GetWindowThreadProcessId(cur, Some(&mut pid));
+ let attached = cur_tid != 0 && cur_tid != my_tid;
+ if attached {
+ let _ = AttachThreadInput(my_tid, cur_tid, true);
+ }
+ let _ = SetForegroundWindow(target);
+ if attached {
+ let _ = AttachThreadInput(my_tid, cur_tid, false);
+ }
+ GetForegroundWindow() == target
+}
+
+/// RAII guard that momentarily drops the system foreground-lock timeout so a
+/// non-UIAccess process can `SetForegroundWindow`, then restores the user's
+/// original value on drop. The change is **in-memory only** β€” `fWinIni` is 0,
+/// so it is NOT written to the user profile (no `SPIF_UPDATEINIFILE`) and never
+/// persists past this guard. Required on machines whose foreground-lock is
+/// maxed (`SPI_GETFOREGROUNDLOCKTIMEOUT` large), which otherwise denies the
+/// raise an occluded WPF window needs.
+struct ForegroundLockGuard {
+ prev: u32,
+ active: bool,
+}
+
+impl ForegroundLockGuard {
+ unsafe fn disable() -> Self {
+ let flags = SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0);
+ let mut prev: u32 = 0;
+ let got = SystemParametersInfoW(
+ SPI_GETFOREGROUNDLOCKTIMEOUT, 0,
+ Some(&mut prev as *mut _ as *mut c_void), flags,
+ )
+ .is_ok();
+ if got && prev != 0 {
+ // value goes in pvParam for this action; 0 = no lock.
+ let _ = SystemParametersInfoW(SPI_SETFOREGROUNDLOCKTIMEOUT, 0, None, flags);
+ }
+ Self { prev, active: got && prev != 0 }
+ }
+}
+
+impl Drop for ForegroundLockGuard {
+ fn drop(&mut self) {
+ if self.active {
+ unsafe {
+ let _ = SystemParametersInfoW(
+ SPI_SETFOREGROUNDLOCKTIMEOUT, 0,
+ Some(self.prev as usize as *mut c_void),
+ SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0),
+ );
+ }
+ }
+ }
+}
+
+/// Make this process the "last input event" provider so Windows' foreground
+/// lock permits our `SetForegroundWindow`. A non-UIAccess process can normally
+/// only set the foreground if it (or the current foreground) sent the last
+/// input; injecting a synthetic, side-effect-free keystroke (a lone Ctrl tap β€”
+/// no menu activation like Alt, no cursor movement like a mouse event) makes
+/// us that provider for the moment that follows.
+unsafe fn foreground_unlock_keypoke() {
+ let mk = |up: bool| INPUT {
+ r#type: INPUT_KEYBOARD,
+ Anonymous: INPUT_0 {
+ ki: KEYBDINPUT {
+ wVk: VIRTUAL_KEY(0x11), // VK_CONTROL
+ wScan: 0,
+ dwFlags: if up { KEYEVENTF_KEYUP } else { Default::default() },
+ time: 0,
+ dwExtraInfo: 0,
+ },
+ },
+ };
+ let ev = [mk(false), mk(true)];
+ SendInput(&ev, std::mem::size_of::<INPUT>() as i32);
+}
+
+/// Forcefully bring `target` to the foreground β€” beating the foreground lock
+/// even from a non-UIAccess process β€” by combining the AttachThreadInput trick
+/// with the synthetic-input unlock above. Used for WPF, which only processes
+/// injected stylus while it is the active foreground window (so an occluded WPF
+/// must be genuinely raised). Returns whether `target` became foreground.
+unsafe fn force_foreground_hard(target: HWND) -> bool {
+ if GetForegroundWindow() == target {
+ return true;
+ }
+ let my_tid = GetCurrentThreadId();
+ let cur = GetForegroundWindow();
+ let mut pid = 0u32;
+ let cur_tid = GetWindowThreadProcessId(cur, Some(&mut pid));
+ let attached = cur_tid != 0 && cur_tid != my_tid;
+ if attached {
+ let _ = AttachThreadInput(my_tid, cur_tid, true);
+ }
+ foreground_unlock_keypoke();
+ let _ = SetForegroundWindow(target);
+ let _ = SetWindowPos(target, HWND_TOP, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE);
+ if attached {
+ let _ = AttachThreadInput(my_tid, cur_tid, false);
+ }
+ GetForegroundWindow() == target
+}
+
+/// RAII guard that makes a specific target window **unable to become the
+/// foreground/active window** for the duration of an actuation, by adding the
+/// `WS_EX_NOACTIVATE` extended style to its top-level window.
+///
+/// Why this and not a global foreground-lock: our own injected/posted input
+/// (or a UIA-Invoke) legitimizes the target's foreground claim, so even a
+/// maxed `SPI_*FOREGROUNDLOCKTIMEOUT` won't stop the steal. `WS_EX_NOACTIVATE`
+/// is categorical β€” Windows refuses to activate the window *at all* (clicks,
+/// `SetForegroundWindow(self)` from WPF/XAML/Tauri handlers, mouse-activate) β€”
+/// while the window still RECEIVES the click/key. It is per-window (no session
+/// side effects) and reversed on drop. Covers the self-activation that the
+/// EnableWindow/UWP bypass cannot (WPF `UIElement.Focus()`β†’SetForegroundWindow).
+pub struct NoActivateGuard {
+ // Store the handle as an integer so the guard is `Send` and can be held
+ // across `.await` in the async tools.
+ root_addr: isize,
+ prev_exstyle: isize,
+ applied: bool,
+}
+
+impl NoActivateGuard {
+ /// Arm on the top-level (GA_ROOT) ancestor of `hwnd`.
+ pub fn arm(hwnd: HWND) -> Self {
+ unsafe {
+ let root = {
+ let r = GetAncestor(hwnd, GA_ROOT);
+ if r.0.is_null() { hwnd } else { r }
+ };
+ let prev = GetWindowLongPtrW(root, GWL_EXSTYLE);
+ let want = WS_EX_NOACTIVATE.0 as isize;
+ // Apply WS_EX_NOACTIVATE if not already set (prev can be 0, so don't
+ // gate on it β€” we just need to check the bit and set it if absent).
+ let applied = (prev & want) == 0 && {
+ SetWindowLongPtrW(root, GWL_EXSTYLE, prev | want);
+ // Confirm it took (cross-process SetWindowLongPtr can be denied
+ // by UIPI on higher-integrity targets).
+ (GetWindowLongPtrW(root, GWL_EXSTYLE) & want) != 0
+ };
+ Self { root_addr: root.0 as isize, prev_exstyle: prev, applied }
+ }
+ }
+}
+
+impl Drop for NoActivateGuard {
+ fn drop(&mut self) {
+ if self.applied {
+ unsafe {
+ let _ = SetWindowLongPtrW(HWND(self.root_addr as *mut _), GWL_EXSTYLE, self.prev_exstyle);
+ }
+ }
+ }
+}
+
+/// PEN_FLAG_BARREL (winuser.h) β€” pen barrel button held == secondary (right)
+/// button. `penFlags` is a raw u32 in the bindings, so use the literal.
+const PEN_FLAG_BARREL: u32 = 0x00000001;
+
+const CLOAK_SIZE: u32 = std::mem::size_of::<BOOL>() as u32;
+
+/// Restore the user's window to the top of the visible z-order WITHOUT
+/// activating it. `SWP_NOACTIVATE` sends no `WM_ACTIVATE`/`WM_MOUSEACTIVATE`
+/// to either window, so this can never block on a busy target's activation
+/// handler (the deadlock that `AttachThreadInput` + `SetForegroundWindow`
+/// risks against a webview that's mid-click). It is also not gated by the
+/// foreground-lock. Best-effort, instant, hang-free.
+unsafe fn restore_z_top(user_win: HWND) {
+ let _ = SetWindowPos(
+ user_win,
+ HWND_TOP,
+ 0,
+ 0,
+ 0,
+ 0,
+ SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,
+ );
+}
+
+/// Put `win` into / out of the always-on-top (topmost) band WITHOUT activating
+/// it. `SWP_NOACTIVATE` means no focus/foreground change. The topmost band sits
+/// above ALL normal windows β€” including an *active* occluder β€” which `HWND_TOP`
+/// alone does not guarantee for a non-activated (esp. `WS_EX_NOACTIVATE`)
+/// window. Used to make a blocked injection target win the coordinate hit-test.
+unsafe fn set_topmost(win: HWND, on: bool) {
+ let after = if on { HWND_TOPMOST } else { HWND_NOTOPMOST };
+ let _ = SetWindowPos(win, after, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE);
+}
+
+unsafe fn set_cloak(h: HWND, on: bool) -> bool {
+ let v: BOOL = if on { TRUE } else { FALSE };
+ DwmSetWindowAttribute(h, DWMWA_CLOAK, &v as *const _ as *const c_void, CLOAK_SIZE).is_ok()
+}
+
+/// RAII guard that lands coordinate-routed injection on an occluded background
+/// target without stealing focus.
+///
+/// Coordinate injection (pen/touch) is delivered to the TOP-MOST **visible**
+/// window at the screen point β€” and a DWM-cloaked window is *excluded* from
+/// hit-testing (verified: injection over a cloaked target lands on the occluder
+/// instead). So we cannot hide the target; to drive it when it's blocked we
+/// briefly raise it to the top of the z-order on `arm` β€” with `SWP_NOACTIVATE`,
+/// so the user's window keeps focus/foreground (no activation, no input-queue
+/// attach) β€” and on `Drop` restore the user's window to the top. The target is
+/// visible on top only for the few milliseconds of the actuation.
+struct ZorderGuard {
+ prev_fg: HWND,
+ target: HWND,
+ raised: bool,
+}
+
+impl ZorderGuard {
+ unsafe fn arm(target: HWND) -> Self {
+ let prev_fg = GetForegroundWindow();
+ // TODO: Check if target is actually occluded (WindowFromPoint over its
+ // client rect) before raising, and preserve its original topmost state
+ // (via GetWindowLongPtr/WS_EX_TOPMOST) so drop() can restore it instead
+ // of unconditionally demoting. Current behavior: raise any non-foreground
+ // target into topmost band (works for common case, but loses original z
+ // and raises even when not occluded).
+ let raised = !target.0.is_null() && target != prev_fg;
+ if raised {
+ set_topmost(target, true);
+ }
+ Self { prev_fg, target, raised }
+ }
+}
+
+impl Drop for ZorderGuard {
+ fn drop(&mut self) {
+ unsafe {
+ if self.raised {
+ // Drop the target back out of the topmost band, then re-stack the
+ // user's window on top (hang-free, no activation messages).
+ set_topmost(self.target, false);
+ if !self.prev_fg.0.is_null() && self.prev_fg != self.target {
+ restore_z_top(self.prev_fg);
+ }
+ }
+ }
+ }
+}
+
+/// One down→up **pen** tap at screen `(sx, sy)`. When `barrel` is set the pen's
+/// barrel button is held for the contact, which the system maps to a secondary
+/// (right) click β€” both for `WM_POINTER`-aware apps (Chromium/WPF/UWP) and via
+/// pen→mouse promotion for legacy Win32. A fresh synthetic pen device is
+/// created and destroyed per tap (right/middle clicks are rare).
+fn pen_tap(sx: i32, sy: i32, barrel: bool) -> Result<()> {
+ unsafe {
+ let dev = CreateSyntheticPointerDevice(PT_PEN, 1, POINTER_FEEDBACK_DEFAULT)
+ .map_err(|e| anyhow::anyhow!("CreateSyntheticPointerDevice(PEN): {e}"))?;
+ let pen_flags = if barrel { PEN_FLAG_BARREL } else { 0 };
+ let mk = |flags| POINTER_TYPE_INFO {
+ r#type: PT_PEN,
+ Anonymous: POINTER_TYPE_INFO_0 {
+ penInfo: POINTER_PEN_INFO {
+ pointerInfo: POINTER_INFO {
+ pointerType: PT_PEN,
+ pointerId: 0,
+ pointerFlags: flags,
+ sourceDevice: HANDLE::default(),
+ hwndTarget: HWND::default(),
+ ptPixelLocation: POINT { x: sx, y: sy },
+ ..Default::default()
+ },
+ penFlags: pen_flags,
+ penMask: 0,
+ pressure: 512,
+ rotation: 0,
+ tiltX: 0,
+ tiltY: 0,
+ },
+ },
+ };
+ let down = mk(POINTER_FLAG_DOWN | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT);
+ let r1 = InjectSyntheticPointerInput(dev, &[down]);
+ sleep(Duration::from_millis(25));
+ let up = mk(POINTER_FLAG_UP);
+ let r2 = InjectSyntheticPointerInput(dev, &[up]);
+ let _ = DestroySyntheticPointerDevice(dev);
+ r1.and(r2).map_err(|e| anyhow::anyhow!("InjectSyntheticPointerInput(pen): {e}"))?;
+ }
+ Ok(())
+}
+
+/// Inject a click at **screen** coordinates `(sx, sy)`, routed by the system to
+/// whatever window is under that point β€” without a foreground swap and without
+/// moving the user's cursor. The target is cloaked for the duration so any
+/// click-activation raise stays invisible, then the user's foreground is
+/// restored.
+///
+/// - `left` β†’ synthetic-pen primary tap (proven path; promoted to a left click
+/// for non-pointer-aware apps, and accepted directly by Chromium/WPF/UWP).
+/// - `right` β†’ synthetic-pen tap with the barrel button held (secondary click).
+/// - `middle`β†’ unsupported (no clean pointer mapping); returns `Err` so the
+/// caller can fall back to its existing routing / structured error.
+///
+/// We use the same `CreateSyntheticPointerDevice`/`InjectSyntheticPointerInput`
+/// path for both buttons β€” `InjectTouchInput` proved unreliable for left-clicks
+/// on Chromium content (returned errors), whereas synthetic-pen injection lands
+/// reliably and routes by coordinate with no foreground dependency.
+pub fn inject_click_screen(target: u64, sx: i32, sy: i32, count: usize, button: &str) -> Result<()> {
+ if target == 0 {
+ bail!("inject_click_screen: null target window");
+ }
+ let target_h = HWND(target as *mut _);
+ unsafe {
+ if !IsWindow(target_h).as_bool() {
+ bail!("inject_click_screen: invalid or stale target HWND");
+ }
+ }
+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target) {
+ // Higher-integrity target: injection into its queue is blocked too.
+ bail!(msg);
+ }
+
+ let barrel = match button {
+ "left" => false,
+ "right" => true,
+ other => bail!("background injection supports left/right buttons only (got {other:?})"),
+ };
+
+ // Make the target categorically non-activatable for the click (so neither
+ // click-activation nor a self-SetForegroundWindow can steal foreground),
+ // and hide/restore any residual z-order via the cloak/SWP guard.
+ let _noact = NoActivateGuard::arm(target_h);
+ let _guard = unsafe { ZorderGuard::arm(target_h) };
+ let count = count.max(1);
+ for i in 0..count {
+ pen_tap(sx, sy, barrel)?;
+ if i + 1 < count {
+ sleep(Duration::from_millis(70));
+ }
+ }
+ // _guard drops here: restore the user's foreground + uncloak target.
+ Ok(())
+}
+
+/// One pen press-drag-release from screen `(sx0,sy0)` to `(sx1,sy1)`, with
+/// `steps` interpolated in-contact UPDATE points between the down and the up.
+/// A single synthetic pen device is created for the whole stroke. The barrel
+/// button is held when `barrel` is set (secondary-button drag).
+fn pen_drag(sx0: i32, sy0: i32, sx1: i32, sy1: i32, steps: usize, barrel: bool) -> Result<()> {
+ unsafe {
+ let dev = CreateSyntheticPointerDevice(PT_PEN, 1, POINTER_FEEDBACK_DEFAULT)
+ .map_err(|e| anyhow::anyhow!("CreateSyntheticPointerDevice(PEN): {e}"))?;
+ let pen_flags = if barrel { PEN_FLAG_BARREL } else { 0 };
+ let mk = |flags, x: i32, y: i32| POINTER_TYPE_INFO {
+ r#type: PT_PEN,
+ Anonymous: POINTER_TYPE_INFO_0 {
+ penInfo: POINTER_PEN_INFO {
+ pointerInfo: POINTER_INFO {
+ pointerType: PT_PEN,
+ pointerId: 0,
+ pointerFlags: flags,
+ sourceDevice: HANDLE::default(),
+ hwndTarget: HWND::default(),
+ ptPixelLocation: POINT { x, y },
+ ..Default::default()
+ },
+ penFlags: pen_flags,
+ penMask: 0,
+ pressure: 512,
+ rotation: 0,
+ tiltX: 0,
+ tiltY: 0,
+ },
+ },
+ };
+ // Press at the start.
+ let down = mk(POINTER_FLAG_DOWN | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT, sx0, sy0);
+ let mut res = InjectSyntheticPointerInput(dev, &[down]);
+ // Interpolated in-contact moves so frameworks that gate drag-tracking on
+ // motion (rather than a single down→up) see a continuous stroke.
+ let steps = steps.max(1);
+ for i in 1..=steps {
+ sleep(Duration::from_millis(8));
+ let t = i as f64 / steps as f64;
+ let x = sx0 + ((sx1 - sx0) as f64 * t).round() as i32;
+ let y = sy0 + ((sy1 - sy0) as f64 * t).round() as i32;
+ let mv = mk(POINTER_FLAG_UPDATE | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT, x, y);
+ res = res.and(InjectSyntheticPointerInput(dev, &[mv]));
+ }
+ // Release at the end.
+ sleep(Duration::from_millis(8));
+ let up = mk(POINTER_FLAG_UP, sx1, sy1);
+ res = res.and(InjectSyntheticPointerInput(dev, &[up]));
+ let _ = DestroySyntheticPointerDevice(dev);
+ res.map_err(|e| anyhow::anyhow!("InjectSyntheticPointerInput(pen drag): {e}"))?;
+ }
+ Ok(())
+}
+
+/// A **persistent** synthetic touch digitizer, created once and never
+/// destroyed. This is load-bearing: a transient (per-stroke) device is gone
+/// before WPF's WISP stylus stack can bind to it, so the OS falls back to
+/// legacy touch→mouse promotion — which drags the user's cursor to the
+/// contact. A *standing* digitizer is enumerated as a real tablet, so WPF (and
+/// other stylus/pointer-aware frameworks) consume the contact as touch/stylus
+/// and promote it to mouse INTERNALLY, without the OS moving the system cursor.
+static TOUCH_DEV: Mutex<isize> = Mutex::new(0);
+
+/// One **touch** press-drag-release from screen `(sx0,sy0)` to `(sx1,sy1)` with
+/// `steps` interpolated in-contact moves, via the persistent [`TOUCH_DEV`].
+/// Unlike a pen (an absolute *cursor* device β€” injecting one drags the user's
+/// mouse pointer along), a touch contact from a standing digitizer is consumed
+/// as touch/stylus and does NOT move the user's cursor. Serialized on the
+/// single shared device (one stroke at a time across all sessions).
+fn touch_drag(sx0: i32, sy0: i32, sx1: i32, sy1: i32, steps: usize) -> Result<()> {
+ let mut dev_guard = TOUCH_DEV.lock().unwrap_or_else(|e| e.into_inner());
+ unsafe {
+ // A non-pointer-aware window (WPF) makes the OS promote the PRIMARY touch
+ // contact to a mouse event, which drags the system cursor to the contact
+ // β€” and the OS gates delivery on the cursor actually reaching it, so the
+ // move can't be prevented from a background process (pinning/clipping the
+ // cursor just drops the input). What we CAN do is snap the cursor back to
+ // exactly where the user left it the instant the stroke ends, so the net
+ // displacement is zero and (with a fast, few-step stroke) the excursion
+ // is a brief flick rather than a sustained drag. Pointer-aware targets
+ // (Chromium) never promote, so the cursor never moves and this restore is
+ // a harmless no-op.
+ let mut cpos = POINT::default();
+ let have_cpos = GetCursorPos(&mut cpos).is_ok();
+ let dev = if *dev_guard != 0 {
+ HSYNTHETICPOINTERDEVICE(*dev_guard as *mut c_void)
+ } else {
+ let d = CreateSyntheticPointerDevice(PT_TOUCH, 1, POINTER_FEEDBACK_DEFAULT)
+ .map_err(|e| anyhow::anyhow!("CreateSyntheticPointerDevice(TOUCH): {e}"))?;
+ *dev_guard = d.0 as isize;
+ d
+ };
+ let mk = |flags, x: i32, y: i32| POINTER_TYPE_INFO {
+ r#type: PT_TOUCH,
+ Anonymous: POINTER_TYPE_INFO_0 {
+ touchInfo: POINTER_TOUCH_INFO {
+ pointerInfo: POINTER_INFO {
+ pointerType: PT_TOUCH,
+ pointerId: 0,
+ pointerFlags: flags,
+ sourceDevice: HANDLE::default(),
+ hwndTarget: HWND::default(),
+ ptPixelLocation: POINT { x, y },
+ ..Default::default()
+ },
+ touchFlags: 0,
+ touchMask: 0,
+ rcContact: RECT { left: x - 2, top: y - 2, right: x + 2, bottom: y + 2 },
+ rcContactRaw: RECT { left: x - 2, top: y - 2, right: x + 2, bottom: y + 2 },
+ orientation: 0,
+ pressure: 512,
+ },
+ },
+ };
+ // Fast stroke: line-tool canvases only need down→up (the segment is the
+ // straight line between them), so a few in-contact frames with a tiny
+ // dwell is plenty β€” and the shorter the stroke, the briefer the cursor
+ // excursion before we snap it back.
+ let down = mk(POINTER_FLAG_DOWN | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT, sx0, sy0);
+ let mut res = InjectSyntheticPointerInput(dev, &[down]);
+ let steps = steps.clamp(1, 3);
+ for i in 1..=steps {
+ sleep(Duration::from_millis(2));
+ let t = i as f64 / steps as f64;
+ let x = sx0 + ((sx1 - sx0) as f64 * t).round() as i32;
+ let y = sy0 + ((sy1 - sy0) as f64 * t).round() as i32;
+ let mv = mk(POINTER_FLAG_UPDATE | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT, x, y);
+ res = res.and(InjectSyntheticPointerInput(dev, &[mv]));
+ }
+ sleep(Duration::from_millis(2));
+ let up = mk(POINTER_FLAG_UP, sx1, sy1);
+ res = res.and(InjectSyntheticPointerInput(dev, &[up]));
+ // Snap the cursor back to where the user left it. The OS processes the
+ // promoted mouse messages slightly after injection, so a single restore
+ // right after the `up` can be overrun by that late move β€” settle briefly,
+ // then restore, and restore once more to win the race. No-op for
+ // pointer-aware targets (Chromium) that never moved the cursor.
+ if have_cpos {
+ let _ = SetCursorPos(cpos.x, cpos.y);
+ sleep(Duration::from_millis(12));
+ let _ = SetCursorPos(cpos.x, cpos.y);
+ }
+ // device intentionally NOT destroyed β€” see TOUCH_DEV.
+ res.map_err(|e| anyhow::anyhow!("InjectSyntheticPointerInput(touch drag): {e}"))?;
+ }
+ Ok(())
+}
+
+/// Inject a press-drag-release at **screen** coordinates, routed by the system
+/// to whatever window is under the path β€” without a foreground swap and without
+/// moving the user's cursor. This is the background fallback for canvases whose
+/// content (Chromium/WPF/GTK) silently drops a PostMessage drag: synthetic-pen
+/// input arrives through the system input queue and is accepted directly
+/// (Chromium/WPF) or promoted to mouse for legacy Win32. `left` β†’ primary
+/// stroke, `right` β†’ barrel-held secondary stroke; the target is held
+/// non-activatable + cloaked so any transient raise stays invisible.
+pub fn inject_drag_screen(
+ target: u64,
+ sx0: i32,
+ sy0: i32,
+ sx1: i32,
+ sy1: i32,
+ steps: usize,
+ button: &str,
+) -> Result<()> {
+ if target == 0 {
+ bail!("inject_drag_screen: null target window");
+ }
+ let target_h = HWND(target as *mut _);
+ unsafe {
+ if !IsWindow(target_h).as_bool() {
+ bail!("inject_drag_screen: invalid or stale target HWND");
+ }
+ }
+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target) {
+ bail!(msg);
+ }
+ let barrel = match button {
+ "left" => false,
+ "right" => true,
+ other => bail!("background injection supports left/right buttons only (got {other:?})"),
+ };
+ let prev_fg = unsafe { GetForegroundWindow() };
+ // Left drag β†’ touch contact (coordinate-routed). Right/barrel drag has no
+ // touch equivalent, so fall back to a pen (rare).
+ let stroke = |()| if barrel { pen_drag(sx0, sy0, sx1, sy1, steps, true) } else { touch_drag(sx0, sy0, sx1, sy1, steps) };
+
+ // WPF (Wisp input) only PROCESSES injected stylus while it is the ACTIVE
+ // foreground window β€” raising it in z while it stays inactive is not enough
+ // (verified by RE). So for WPF we must briefly activate it (a visible raise
+ // + focus, which active⇒foreground⇒topmost also un-occludes), inject, then
+ // restore the user's foreground. Other coordinate-injection targets
+ // (Chromium/GTK) are pointer-aware and process injection in the background,
+ // so we hold them non-activatable and only raise them into the topmost band
+ // to win the hit-test when occluded β€” no focus steal.
+ let needs_active = crate::input::dispatch::is_wpf_target_window(target);
+ if needs_active {
+ // Break the no-raise contract for WPF: fully raise+activate it for the
+ // brief moment of the stroke so its Wisp input stack processes the
+ // injected stylus, then restore the user's window. The machine's
+ // foreground-lock is dropped (in-memory only) for this window so the
+ // raise is permitted, and restored immediately afterwards.
+ let _lock = unsafe { ForegroundLockGuard::disable() };
+ unsafe { force_foreground_hard(target_h); }
+ let r = stroke(());
+ unsafe {
+ if !prev_fg.0.is_null() && prev_fg != target_h {
+ force_foreground_hard(prev_fg);
+ }
+ }
+ return r;
+ }
+ // Chromium/GTK: pointer-aware, process injection in the background β€” hold
+ // non-activatable + raise into the topmost band to win the hit-test when
+ // occluded, no focus steal.
+ let r = {
+ let _noact = NoActivateGuard::arm(target_h);
+ let _guard = unsafe { ZorderGuard::arm(target_h) };
+ stroke(())
+ };
+ unsafe {
+ if !prev_fg.0.is_null() && prev_fg != target_h {
+ force_foreground_attached(prev_fg);
+ }
+ }
+ r
+}
+
+/// Send `key` (+ optional `modifiers`) to a **background** target via the
+/// system input queue, with the target cloaked so the brief focus it needs
+/// never shows as a visible raise.
+///
+/// Keyboard input β€” unlike mouse β€” has no coordinate routing: synthesized keys
+/// go to the *focused* window of the foreground queue, so the target must hold
+/// focus to receive a SendInput accelerator (Ctrl+S, Ctrl+A) that frameworks
+/// detect via `GetKeyState`/`TranslateAccelerator`.
+///
+/// Capability-first contract: the keystroke MUST be delivered. We make a
+/// best-effort to preserve the background UX (cloak the target so its brief
+/// foreground stint is hidden, restore the user's foreground after), but we do
+/// NOT abandon the action to keep the UX:
+/// 1. Cloak the target (DWM) so any raise is invisible.
+/// 2. Bring it foreground via the AttachThreadInput trick (beats the
+/// foreground-lock even without UIAccess) and SendInput the combo, so
+/// `GetKeyState` updates and the accelerator actually fires.
+/// 3. If focus genuinely can't be obtained, fall back to PostMessage so the
+/// key still reaches the window (best-effort; may miss GetKeyState-gated
+/// accelerators, but never silently drops the action).
+/// 4. Restore the user's foreground and uncloak.
+pub fn inject_key_cloaked(target: u64, key: &str, modifiers: &[&str]) -> Result<()> {
+ let target_h = HWND(target as *mut _);
+ if target_h.0.is_null() {
+ bail!("invalid target hwnd");
+ }
+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target) {
+ bail!(msg);
+ }
+
+ let _serial = fg_serialize(); // one cloaked-foreground op at a time (1s ceiling)
+ let prev_fg = unsafe { GetForegroundWindow() };
+ let cloaked = unsafe { target_h != prev_fg && set_cloak(target_h, true) };
+ let got_fg = unsafe { force_foreground_attached(target_h) };
+
+ let result = if got_fg {
+ // Target is foreground (cloaked): send_key_synthesized's own
+ // SetForegroundWindow is a no-op success; SendInput updates GetKeyState
+ // so the accelerator fires.
+ crate::input::send_key_synthesized(target, key, modifiers)
+ } else {
+ // Couldn't focus the target even with the attach trick β€” deliver
+ // best-effort via PostMessage rather than dropping the action.
+ crate::input::post_key(target, key, modifiers)
+ };
+
+ unsafe {
+ if !prev_fg.0.is_null() && prev_fg != target_h {
+ force_foreground_attached(prev_fg);
+ }
+ if cloaked {
+ let _ = set_cloak(target_h, false);
+ }
+ }
+ result
+}
+
+/// Type `text` into a **background** target via real SendInput Unicode
+/// keystrokes, cloaked so the brief focus is hidden, then restore foreground.
+///
+/// For targets that ignore a posted `WM_CHAR` (WPF, whose TextBox only consumes
+/// real keyboard input routed through its own input manager), `post_type_text`
+/// silently does nothing. This delivers genuine `KEYEVENTF_UNICODE` keystrokes
+/// to the focused control while the target briefly (and invisibly) holds focus.
+/// Capability-first: the text is delivered; the focus flicker is hidden and the
+/// user's foreground restored. Caller should focus the field first (a prior
+/// background click on it) so the keystrokes land in the right control.
+pub fn inject_text_cloaked(target: u64, text: &str) -> Result<()> {
+ let target_h = HWND(target as *mut _);
+ if target_h.0.is_null() {
+ bail!("invalid target hwnd");
+ }
+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target) {
+ bail!(msg);
+ }
+ let _serial = fg_serialize(); // one cloaked-foreground op at a time (1s ceiling)
+ let prev_fg = unsafe { GetForegroundWindow() };
+ let cloaked = unsafe { target_h != prev_fg && set_cloak(target_h, true) };
+ let got_fg = unsafe { force_foreground_attached(target_h) };
+
+ let result = if got_fg {
+ unsafe { send_unicode(text) }
+ } else {
+ crate::input::post_type_text(target, text)
+ };
+
+ unsafe {
+ if !prev_fg.0.is_null() && prev_fg != target_h {
+ force_foreground_attached(prev_fg);
+ }
+ if cloaked {
+ let _ = set_cloak(target_h, false);
+ }
+ }
+ result
+}
+
+fn key_unicode(unit: u16, up: bool) -> INPUT {
+ let mut flags = KEYEVENTF_UNICODE;
+ if up {
+ flags |= KEYEVENTF_KEYUP;
+ }
+ INPUT {
+ r#type: INPUT_KEYBOARD,
+ Anonymous: INPUT_0 {
+ ki: KEYBDINPUT {
+ wVk: VIRTUAL_KEY(0),
+ wScan: unit,
+ dwFlags: flags,
+ time: 0,
+ dwExtraInfo: 0,
+ },
+ },
+ }
+}
+
+unsafe fn send_unicode(text: &str) -> Result<()> {
+ let mut ev: Vec<INPUT> = Vec::with_capacity(text.len() * 2);
+ for u in text.encode_utf16() {
+ ev.push(key_unicode(u, false));
+ ev.push(key_unicode(u, true));
+ }
+ if ev.is_empty() {
+ return Ok(());
+ }
+ let sent = SendInput(&ev, std::mem::size_of::<INPUT>() as i32);
+ if sent as usize != ev.len() {
+ bail!("SendInput typed only {sent} of {} key events", ev.len());
+ }
+ Ok(())
+}
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
index bf7b1c3f49..d1bb662f88 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs
@@ -12,7 +12,9 @@
pub mod mouse;
pub mod keyboard;
pub mod dispatch;
+pub mod inject;
+pub use inject::{inject_click_screen, inject_key_cloaked, inject_text_cloaked, NoActivateGuard};
pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized};
pub use keyboard::{
is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
index 8a7cbc2f47..ae447fab19 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs
@@ -166,6 +166,62 @@ pub fn post_drag(
Ok(())
}
+/// Press-drag-release via PostMessage, resolving the **deepest child** at the
+/// drag-start screen point and posting in that child's client coordinates.
+///
+/// `post_drag` (above) posts to the top-level frame, so a child-windowed
+/// control (a WinForms `Panel`, a Win32 child canvas, …) never sees the drag β€”
+/// the frame gets messages over a region it doesn't own and ignores them. This
+/// variant mirrors `post_click`: it hit-tests down to the deepest descendant
+/// under the start point and targets that HWND for the whole gesture (a drag
+/// stays within one control), with each point converted to the child's own
+/// client space. Endpoints are given in **screen** coordinates.
+pub fn post_drag_screen(
+ root: u64,
+ sx_from: i32,
+ sy_from: i32,
+ sx_to: i32,
+ sy_to: i32,
+ duration_ms: u64,
+ steps: usize,
+ button: &str,
+) -> Result<()> {
+ let root_hwnd = HWND(root as *mut _);
+ let (target, c_from) = deepest_child(root_hwnd, POINT { x: sx_from, y: sy_from });
+ let mut c_to = POINT { x: sx_to, y: sy_to };
+ unsafe { let _ = ScreenToClient(target, &mut c_to); }
+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target.0 as u64) {
+ anyhow::bail!(msg);
+ }
+ let (down_msg, up_msg, mk_flag) = match button {
+ "right" => (WM_RBUTTONDOWN, WM_RBUTTONUP, MK_RBUTTON),
+ "middle" => (WM_MBUTTONDOWN, WM_MBUTTONUP, MK_MBUTTON),
+ _ => (WM_LBUTTONDOWN, WM_LBUTTONUP, MK_LBUTTON),
+ };
+ let wparam = WPARAM(mk_flag as usize);
+ let steps = steps.max(1);
+ let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };
+ unsafe {
+ // Pre-drag MOUSEMOVE (wParam=0, no buttons down yet) then DOWN at from.
+ PostMessageW(target, WM_MOUSEMOVE, WPARAM(0), make_lparam(c_from.x, c_from.y))?;
+ PostMessageW(target, down_msg, wparam, make_lparam(c_from.x, c_from.y))?;
+ }
+ sleep(Duration::from_millis(CLICK_DELAY_MS));
+ for i in 1..=steps {
+ let t = i as f64 / steps as f64;
+ let ix = c_from.x + ((c_to.x - c_from.x) as f64 * t).round() as i32;
+ let iy = c_from.y + ((c_to.y - c_from.y) as f64 * t).round() as i32;
+ unsafe { PostMessageW(target, WM_MOUSEMOVE, wparam, make_lparam(ix, iy))?; }
+ if step_delay_ms > 0 {
+ sleep(Duration::from_millis(step_delay_ms));
+ }
+ }
+ unsafe {
+ PostMessageW(target, up_msg, WPARAM(0), make_lparam(c_to.x, c_to.y))?;
+ }
+ Ok(())
+}
+
/// Pack two 16-bit integers into a LPARAM (low word = x, high word = y).
fn make_lparam(x: i32, y: i32) -> LPARAM {
LPARAM((((y as u16 as u32) << 16) | (x as u16 as u32)) as isize)
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs
index 6cff851e57..3c057fc3d1 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs
@@ -564,6 +564,35 @@ unsafe extern "system" fn wnd_proc(
.unwrap_or_default()
.as_millis() as u64;
+ // ── Optional overlay-FPS probe ───────────────────────────────────
+ // Set CUA_DRIVER_RS_OVERLAY_FPS_FILE=<path> to append a measured
+ // render-FPS line ~once/sec. Diagnostic only; when the env var is
+ // unset this is a single OnceLock read + branch (no behaviour change).
+ {
+ use std::sync::atomic::{AtomicU64, Ordering::Relaxed};
+ static FPS_PATH: std::sync::OnceLock<Option<String>> = std::sync::OnceLock::new();
+ static FPS_FRAMES: AtomicU64 = AtomicU64::new(0);
+ static FPS_LAST: AtomicU64 = AtomicU64::new(0);
+ if let Some(path) = FPS_PATH.get_or_init(|| std::env::var("CUA_DRIVER_RS_OVERLAY_FPS_FILE").ok()) {
+ let n = FPS_FRAMES.fetch_add(1, Relaxed) + 1;
+ let last = FPS_LAST.load(Relaxed);
+ if last == 0 {
+ FPS_LAST.store(now_ms, Relaxed);
+ } else if now_ms.wrapping_sub(last) >= 1000 {
+ let secs = (now_ms - last) as f64 / 1000.0;
+ let fps = n as f64 / secs.max(1e-3);
+ let cursors = RENDER.lock().ok()
+ .and_then(|g| g.as_ref().map(|m| m.cursors.len())).unwrap_or(0);
+ if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(path) {
+ use std::io::Write;
+ let _ = writeln!(f, "overlay fps={fps:.1} avg_dt_ms={:.1} cursors={cursors}", secs * 1000.0 / n as f64);
+ }
+ FPS_FRAMES.store(0, Relaxed);
+ FPS_LAST.store(now_ms, Relaxed);
+ }
+ }
+ }
+
// ── Drain commands, tick all cursors, composite one pixmap ───────
// Measure real dt from last tick β€” Windows timer resolution defaults
// to 15ms so the hardcoded 8ms ran the animation at half speed.
@@ -572,9 +601,11 @@ unsafe extern "system" fn wnd_proc(
if let Some(map) = guard.as_mut() {
// Drain the channel via get-or-create; track the last-touched
// key so the z-order pin follows the most-recent cursor.
+ let mut drained = 0u32;
if let Ok(rx_guard) = CMD_RX_WIN.try_lock() {
if let Some(ref rx) = *rx_guard {
while let Ok(m) = rx.try_recv() {
+ drained += 1;
if let Some(k) = apply_msg(map, m) {
map.last_active = Some(k);
}
@@ -597,10 +628,39 @@ unsafe extern "system" fn wnd_proc(
}
}
- // Composite every cursor into ONE virtual-screen pixmap.
- // tiny-skia fills are alpha-over, so insertion order =
- // paint/z-order; idle/hidden cursors early-return inside
- // paint_cursor so an idle session costs ~nothing.
+ // Decide where to pin the single overlay window in z.
+ //
+ // The overlay is ONE full-virtual-screen layered window, so it
+ // can occupy only one z-slot. It must sit ABOVE every window a
+ // live cursor is actuating, but NOT above whatever sits above
+ // those (the user's foreground). The right slot is therefore
+ // "just above the HIGHEST-z actuating window": the overlay is
+ // full-screen, so being above the topmost driven window puts it
+ // above all of them (they're all at-or-below it), while still
+ // below anything stacked above them. Pinning above one fixed
+ // window (the old last-active behaviour) instead let any other
+ // driven window stacked above it occlude its cursors β€” the
+ // blink-out. NB: this is a RELATIVE z move (insert above a
+ // specific window), which works from this non-foreground
+ // thread; an absolute HWND_TOP can be refused by the
+ // foreground lock and sink the overlay behind everything.
+ let mut driven: Vec<u64> = Vec::new();
+ for rs in map.cursors.values() {
+ if !rs.core.visible || rs.core.idle_alpha < 0.004 { continue; }
+ if let Some(w) = rs.core.pinned_wid {
+ if !driven.contains(&w) { driven.push(w); }
+ }
+ }
+ let pinned = unsafe { topmost_of(&driven) };
+
+ // Composite every cursor into ONE virtual-screen pixmap and
+ // blit it every frame. (An earlier "skip when idle" gate was
+ // removed: starting/stopping the full-screen UpdateLayeredWindow
+ // as activity comes and goes made all the resting cursors
+ // flicker when any one of them clicked β€” very visible with many
+ // cursors. A steady per-frame blit is flicker-free, and during
+ // playback at least one cursor is almost always active anyway.)
+ let _ = drained;
let w = map.virt_w.max(1) as u32;
let h = map.virt_h.max(1) as u32;
let mut pm = tiny_skia::Pixmap::new(w.max(1), h.max(1))
@@ -615,13 +675,6 @@ unsafe extern "system" fn wnd_proc(
);
}
- // Pin above the most-recently-touched cursor's target.
- let pinned = map
- .last_active
- .as_ref()
- .and_then(|k| map.cursors.get(k))
- .and_then(|rs| rs.core.pinned_wid);
-
(Some(pm), arrived, pinned)
} else {
(None, Vec::new(), None)
@@ -770,6 +823,23 @@ struct WinZOrderEnforcer {
hwnd_isize: isize,
}
+/// Of the given window ids, return the one highest in the current z-order (the
+/// first encountered walking top→bottom), or `None` if none are present. Used to
+/// pick the single window the overlay should pin just above so it covers every
+/// actuating window without rising above whatever sits above them.
+#[cfg(target_os = "windows")]
+unsafe fn topmost_of(ids: &[u64]) -> Option<u64> {
+ use windows::Win32::Foundation::HWND;
+ use windows::Win32::UI::WindowsAndMessaging::{GetTopWindow, GetWindow, GW_HWNDNEXT};
+ if ids.is_empty() { return None; }
+ let mut h = GetTopWindow(None).unwrap_or(HWND(std::ptr::null_mut()));
+ while !h.0.is_null() {
+ if ids.contains(&(h.0 as u64)) { return Some(h.0 as u64); }
+ h = GetWindow(h, GW_HWNDNEXT).unwrap_or(HWND(std::ptr::null_mut()));
+ }
+ ids.first().copied()
+}
+
impl ZOrderEnforcer for WinZOrderEnforcer {
fn reassert(&self, target: Option<u64>) {
#[cfg(target_os = "windows")]
diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
index 0542127abc..b1678d28b4 100644
--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs
@@ -96,6 +96,28 @@ async fn overlay_glide_to(key: &str, sx: f64, sy: f64) {
crate::overlay::send_command(key.to_owned(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });
return;
}
+ // Fixed-duration glide β†’ decouple the click from the render thread. When
+ // `glide_duration_ms > 0` the path completes in exactly that wall-clock time
+ // (see render_state::tick_motion), so instead of `await`-ing the render
+ // thread's arrival oneshot β€” which couples click latency to overlay FPS and
+ // degrades under many concurrent cursors β€” we fire the move and sleep the
+ // known duration. The click then lands a deterministic time after dispatch
+ // regardless of render load, while the cursor still animates visually on the
+ // render thread. Speed-based glides (`== 0`, the default) keep the precise
+ // arrival-await since their duration depends on distance.
+ let motion = crate::overlay::current_motion(key);
+ if motion.glide_duration_ms > 0.0 {
+ crate::overlay::send_command(
+ key.to_owned(),
+ cursor_overlay::OverlayCommand::MoveTo {
+ x: sx,
+ y: sy,
+ end_heading_radians: std::f64::consts::FRAC_PI_4,
+ },
+ );
+ tokio::time::sleep(std::time::Duration::from_millis(motion.glide_duration_ms as u64)).await;
+ return;
+ }
crate::overlay::animate_cursor_to(key.to_owned(), sx, sy).await;
}
use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}};
@@ -1825,6 +1847,19 @@ impl Tool for ClickTool {
let button = args.str_or("button", "left");
let count = args.u64_or("count", 1) as usize;
let dispatch = DispatchMode::from_args(&args);
+ // For every non-foreground click, mark the target window
+ // non-activatable (WS_EX_NOACTIVATE) for the duration so a target that
+ // self-activates in its UIA-Invoke / click handler (WPF
+ // `UIElement.Focus()`, XAML, Tauri/WebView2) CANNOT steal the user's
+ // foreground β€” the window still receives the click. Held for the whole
+ // invoke; a no-op for dispatch:"foreground" (which wants the swap) and
+ // when no window_id was given.
+ let _noact = match (dispatch != DispatchMode::Foreground, hwnd_opt) {
+ (true, Some(h)) => Some(crate::input::NoActivateGuard::arm(
+ windows::Win32::Foundation::HWND(h as *mut _),
+ )),
+ _ => None,
+ };
// Optional `action` arg picks among the actions exposed in the
// accessibility tree. Today this only changes behavior for MSAA
// BUTTONDROPDOWN: `"expand"` clicks the right-edge (dropdown arrow
@@ -2058,14 +2093,22 @@ impl Tool for ClickTool {
}
}
// PostMessage fallback (legacy Win32 + non-Invokable elements).
- // dispatch:"background" refuses the fallback on targets known
- // to silently drop PostMessage clicks (Chromium content, GTK
- // buttons). We surface a tagged error here so the outer match
- // can convert to the structured background_unavailable result.
+ // dispatch:"background" on targets that silently drop PostMessage
+ // clicks (Chromium content, GTK buttons): route through the
+ // universal coordinate-injection actuator (touch injection, no
+ // foreground swap, z-order preserved) so the caller never needs
+ // to know the target is Chromium/GTK and never sees a raise.
+ // Only the structured error remains as a last resort (e.g. a
+ // right-click, which has no clean touch mapping).
if dispatch == DispatchMode::Background
&& crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick)
{
- anyhow::bail!("__CUA_BG_UNAVAILABLE_CLICK__");
+ match crate::input::inject_click_screen(hwnd, cx, cy, count, &btn) {
+ Ok(()) => return Ok(format!(
+ "βœ… Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)."
+ )),
+ Err(_) => anyhow::bail!("__CUA_BG_UNAVAILABLE_CLICK__"),
+ }
}
crate::input::post_click_screen(hwnd, cx, cy, count, &btn)?;
let action_name = match btn.as_str() {
@@ -2173,18 +2216,38 @@ impl Tool for ClickTool {
}
}
- // UIA hit-test didn't land. Decide between PostMessage / SendInput
- // based on dispatch mode.
+ // UIA hit-test didn't land. Decide between PostMessage / injection /
+ // SendInput based on dispatch mode.
//
- // dispatch:"background" β€” refuse to swap foreground. If the target
- // is known to silently drop PostMessage mouse events (Chromium
- // DOM content, GTK button widgets), surface a structured
- // background_unavailable error so the caller can bring_to_front
- // then retry with dispatch:"foreground".
+ // dispatch:"background" (the default) β€” never swap foreground. If the
+ // target silently drops PostMessage mouse events (Chromium DOM
+ // content, GTK button widgets), route through the universal
+ // coordinate-injection actuator: touch injection lands in the system
+ // input queue (so Chromium/Electron/WPF accept it; the OS promotes to
+ // WM_*BUTTON for legacy Win32) WITHOUT SetForegroundWindow, and a
+ // cloak+restore z-order guard keeps the target from visibly raising.
+ // This is what lets a caller "just target the app and play actions"
+ // without knowing whether it's Chromium/GTK/etc. The structured
+ // background_unavailable error only survives as a last resort for
+ // inputs injection can't express (e.g. right/middle clicks).
if dispatch == DispatchMode::Background
&& crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick)
{
- return background_unavailable_error(hwnd, EventKind::MouseClick);
+ let btn2 = btn.clone();
+ let inj = tokio::task::spawn_blocking(move || {
+ crate::input::inject_click_screen(hwnd, sx as i32, sy as i32, count, &btn2)
+ })
+ .await;
+ return match inj {
+ Ok(Ok(())) => {
+ let click_word = match count { 2 => "double-click", 3 => "triple-click", _ => "click" };
+ ToolResult::text(format!(
+ "βœ… Injected {click_word} to pid {pid} at ({sx},{sy}) (background, no foreground swap)."
+ ))
+ }
+ Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
}
// dispatch:"auto" β€” historical heuristic: Chromium targets get
@@ -2356,6 +2419,17 @@ impl Tool for TypeTextTool {
is scoped per (pid, window_id). Pass the same window_id you used in \
`get_window_state`.");
}
+ // Same no-raise guard as click: a XAML/WPF ValuePattern.SetValue handler
+ // calls UIElement.Focus()β†’SetForegroundWindow; WS_EX_NOACTIVATE on the
+ // target makes that a no-op while the value still gets set. Safe because
+ // type_text never uses the SendInput foreground-swap path. No-op for
+ // dispatch:"foreground" and when no window_id was given.
+ let _noact = match (dispatch != DispatchMode::Foreground, hwnd_opt) {
+ (true, Some(h)) => Some(crate::input::NoActivateGuard::arm(
+ windows::Win32::Foundation::HWND(h as *mut _),
+ )),
+ _ => None,
+ };
let _delay_ms = args.u64_or("delay_ms", 30);
let hwnd = match hwnd_opt {
Some(h) => h,
@@ -2419,53 +2493,60 @@ impl Tool for TypeTextTool {
// uses, which we've verified works on modern Notepad / WinUI3.
// Legacy Win32 stays on the PostMessage path so the no-focus-
// steal property is preserved.
- if crate::input::is_xaml_host_hwnd(hwnd) {
- if let Some(idx) = elem_idx {
- let idx = idx as usize;
- let state = self.state.clone();
- let text_for_uia = text.clone();
- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {
- let ptr = state.element_cache.get_element_ptr(pid, hwnd, idx)
- .ok_or_else(|| anyhow::anyhow!(
- "Element {idx} not in cache β€” call get_window_state(pid={pid}, window_id={hwnd}) first."
- ))?;
- use windows::Win32::UI::Accessibility::{
- IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,
- };
- use windows::core::{Interface, BSTR};
- let elem: IUIAutomationElement =
- unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
- let pattern = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId)? };
- std::mem::forget(elem);
+ // ── Automatic routing β€” the caller need not know the framework. ──
+ // 1. With an element_index, try UIA ValuePattern.SetValue first: it works
+ // for WPF / WinForms / UWP / XAML and many web inputs, sets the value
+ // through the accessibility channel (no keystrokes), and the `_noact`
+ // guard blocks any self-foreground β€” so it never raises. Auto-falls-
+ // back to the WM_CHAR path below if the element has no ValuePattern
+ // (most legacy Win32 EDITs consume WM_CHAR fine without focus steal).
+ if let Some(idx) = elem_idx {
+ let idx = idx as usize;
+ let state = self.state.clone();
+ let text_for_uia = text.clone();
+ let set_ok = tokio::task::spawn_blocking(move || -> bool {
+ let Some(ptr) = state.element_cache.get_element_ptr(pid, hwnd, idx) else { return false; };
+ use windows::Win32::UI::Accessibility::{
+ IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,
+ };
+ use windows::core::{Interface, BSTR};
+ let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };
+ let ok = (|| -> anyhow::Result<()> {
+ let pattern = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId) }?;
let vp: IUIAutomationValuePattern = pattern.cast()?;
- unsafe { vp.SetValue(&BSTR::from(text_for_uia.as_str()))? };
+ unsafe { vp.SetValue(&BSTR::from(text_for_uia.as_str())) }?;
Ok(())
- }).await;
- return match result {
- Ok(Ok(())) => ToolResult::text(format!(
- "βœ… Wrote {text_len} char(s) on pid {raw_pid} via UIA ValuePattern \
- (XAML / UWP target, element_index=[{idx}])."
- )),
- Ok(Err(e)) => ToolResult::error(format!("type_text (UIA path): {e}")),
- Err(e) => ToolResult::error(format!("Task error: {e}")),
- };
- } else {
- // XAML target without element_index: PostMessage will silently
- // drop chars. Surface a clear error pointing the agent at the
- // right workflow rather than lying with a "βœ… Typed" message.
- return ToolResult::error(format!(
- "type_text on a modern XAML / UWP target (pid {raw_pid}, hwnd {hwnd}) \
- requires `element_index` β€” its WM_CHAR pipeline ignores PostMessage \
- without keyboard focus. Call `get_window_state(pid={raw_pid}, \
- window_id={hwnd})` to enumerate elements, then re-call \
- `type_text(pid, window_id, element_index, text)`. Or call \
- `set_value(pid, window_id, element_index, value)` directly β€” same \
- UIA backend."
+ })().is_ok();
+ std::mem::forget(elem);
+ ok
+ }).await.unwrap_or(false);
+ if set_ok {
+ return ToolResult::text(format!(
+ "βœ… Wrote {text_len} char(s) on pid {raw_pid} via UIA ValuePattern (element_index=[{idx}])."
));
}
+ // ValuePattern unavailable β†’ fall through to the WM_CHAR path.
}
- // Legacy Win32 path β€” PostMessage WM_CHAR, no focus steal.
+ // 2. No element_index on a WPF target: WM_CHAR is dropped and there's no
+ // element to SetValue, so deliver real keystrokes via the cloaked-
+ // focus path (capability-first; the brief focus is hidden, foreground
+ // restored). Supplying an element_index (path 1) is preferred and
+ // never raises.
+ if elem_idx.is_none() && crate::input::dispatch::is_wpf_target_window(hwnd) {
+ drop(_noact);
+ let text2 = text.clone();
+ let r = tokio::task::spawn_blocking(move || crate::input::inject_text_cloaked(hwnd, &text2)).await;
+ return match r {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "βœ… Typed {text_len} char(s) on pid {raw_pid} via SendInput (WPF, cloaked focus)."
+ )),
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
+ }
+
+ // 3. Legacy Win32 / GDI / Chromium-IME β€” PostMessage WM_CHAR, no focus steal.
let result = tokio::task::spawn_blocking(move || {
crate::input::post_type_text(hwnd, &text)
}).await;
@@ -2553,7 +2634,24 @@ impl Tool for PressKeyTool {
if dispatch == DispatchMode::Background
&& crate::input::dispatch::would_be_silently_dropped(hwnd, event_kind)
{
- return background_unavailable_error(hwnd, event_kind);
+ // Universal background keyboard actuator: cloaked focus + SendInput,
+ // so TranslateAccelerator-based shortcuts (VCL/classic Win32) and
+ // Chromium key-combos fire without a visible foreground raise. The
+ // structured error only survives when focus can't be obtained
+ // (foreground-lock + no UIAccess β†’ route via the uia worker).
+ let key_i = key.clone();
+ let mods_i: Vec<String> = mods.clone();
+ let inj = tokio::task::spawn_blocking(move || {
+ let m: Vec<&str> = mods_i.iter().map(String::as_str).collect();
+ crate::input::inject_key_cloaked(hwnd, &key_i, &m)
+ }).await;
+ return match inj {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "βœ… Sent {key_display} on pid {raw_pid} (background; cloaked focus if needed)."
+ )),
+ Ok(Err(_)) => background_unavailable_error(hwnd, event_kind),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
}
// Foreground: send_key_synthesized takes the SetForegroundWindow path.
if dispatch == DispatchMode::Foreground {
@@ -2783,7 +2881,22 @@ impl Tool for HotkeyTool {
if dispatch == DispatchMode::Background
&& crate::input::dispatch::would_be_silently_dropped(hwnd, event_kind)
{
- return background_unavailable_error(hwnd, event_kind);
+ // Universal background keyboard actuator (see press_key): cloaked
+ // focus + SendInput so VCL/Chromium accelerators fire without a
+ // visible raise. Structured error only if focus can't be obtained.
+ let key_i = key.clone();
+ let mods_i: Vec<String> = mods.clone();
+ let inj = tokio::task::spawn_blocking(move || {
+ let m: Vec<&str> = mods_i.iter().map(String::as_str).collect();
+ crate::input::inject_key_cloaked(hwnd, &key_i, &m)
+ }).await;
+ return match inj {
+ Ok(Ok(())) => ToolResult::text(format!(
+ "βœ… Pressed {key_display} on pid {raw_pid} (background; cloaked focus if needed)."
+ )),
+ Ok(Err(_)) => background_unavailable_error(hwnd, event_kind),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
}
// dispatch:"foreground" β€” explicit SendInput swap (the path that
// unblocks TranslateAccelerator-style apps). Auto mode preserves the
@@ -2876,6 +2989,17 @@ impl Tool for SetValueTool {
None => return ToolResult::error("Missing required string field value."),
};
+ // No-raise guard: a WPF/XAML automation peer calls UIElement.Focus() β†’
+ // SetForegroundWindow during ValuePattern.SetValue. WS_EX_NOACTIVATE on
+ // the target makes that a no-op while the value is still set, so a
+ // background SetValue can't steal the user's foreground. Held across the
+ // whole write. (No-op for dispatch:"foreground".)
+ let _noact = if crate::input::dispatch::DispatchMode::from_args(&args)
+ != crate::input::dispatch::DispatchMode::Foreground
+ {
+ Some(crate::input::NoActivateGuard::arm(windows::Win32::Foundation::HWND(hwnd as *mut _)))
+ } else { None };
+
// Glide the agent cursor onto the target element before writing its
// value, so a value write gets the same visual feedback as a click β€”
// the viewer can see *where* the agent is acting. No-op when the
@@ -3546,7 +3670,7 @@ impl Tool for DragTool {
})
}
async fn invoke(&self, args: Value) -> ToolResult {
- use crate::input::dispatch::{DispatchMode, EventKind, background_unavailable_error};
+ use crate::input::dispatch::{DispatchMode, EventKind};
// Swift error wording 1:1.
let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {
Some(p) => p,
@@ -3609,11 +3733,43 @@ impl Tool for DragTool {
let (sx_from, sy_from) = bitmap_to_screen(hwnd, from_x as i32, from_y as i32);
let (sx_to, sy_to) = bitmap_to_screen(hwnd, to_x as i32, to_y as i32);
- // dispatch:"background" β€” refuse if PostMessage drag would silently drop.
+ // dispatch:"background" β€” if a PostMessage drag would silently drop
+ // (Chromium/WPF/GTK canvas content reads mouse from the system input
+ // queue, not the per-window queue), fall back to coordinate-routed
+ // synthetic-pen drag injection instead of refusing. No foreground swap,
+ // no cursor move; the target is held non-activatable + cloaked for the
+ // stroke (mirrors the click pen path).
if dispatch == DispatchMode::Background
&& crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick)
{
- return background_unavailable_error(hwnd, EventKind::MouseClick);
+ let target = hwnd;
+ let btn = button.clone();
+ pin_overlay_above(&cursor_key, hwnd);
+ overlay_glide_to(&cursor_key, sx_from as f64, sy_from as f64).await;
+ crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
+ x: sx_from as f64, y: sy_from as f64,
+ });
+ let inj = tokio::task::spawn_blocking(move || {
+ crate::input::inject::inject_drag_screen(
+ target, sx_from, sy_from, sx_to, sy_to, steps.max(8), &btn,
+ )
+ })
+ .await;
+ return match inj {
+ Ok(Ok(())) => {
+ overlay_glide_to(&cursor_key, sx_to as f64, sy_to as f64).await;
+ crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {
+ x: sx_to as f64, y: sy_to as f64,
+ });
+ ToolResult::text(format!(
+ "βœ… Sent drag via synthetic-pen injection on pid {raw_pid} \
+ from screen ({sx_from},{sy_from}) β†’ ({sx_to},{sy_to}) \
+ (dispatch:background, PostMessage would have been dropped)."
+ ))
+ }
+ Ok(Err(e)) => ToolResult::error(e.to_string()),
+ Err(e) => ToolResult::error(format!("Task error: {e}")),
+ };
}
// dispatch:"foreground" β€” SendInput-based drag. Required for WPF
// Slider thumbs (and any framework that polls GetKeyState during
@@ -3667,10 +3823,13 @@ impl Tool for DragTool {
let button_c = button.clone();
let result = tokio::task::spawn_blocking(move || {
- crate::input::mouse::post_drag(
+ // Screen-coord, deepest-child variant: routes the gesture to the
+ // child control under the start point (e.g. a WinForms Panel),
+ // not the top-level frame that would ignore it.
+ crate::input::mouse::post_drag_screen(
hwnd,
- from_x as i32, from_y as i32,
- to_x as i32, to_y as i32,
+ sx_from, sy_from,
+ sx_to, sy_to,
duration_ms, steps, &button_c,
)
}).await;
@@ -3877,7 +4036,7 @@ impl Tool for SetAgentCursorMotionTool {
Motion curve (Bezier):\n\
- arc_size: perpendicular deflection as fraction of path length [0,1]. Default 0.25\n\
- spring: settle damping [0.3,1.0]; 1.0=no overshoot. Default 0.72\n\
- - glide_duration_ms: flight duration per move [50,5000]. Default 160\n\
+ - glide_duration_ms: fixed flight duration per move [50,5000]; omit for speed-based (the default)\n\
- dwell_after_click_ms: pause after click ripple [0,5000]. Default 80\n\
- idle_hide_ms: auto-hide delay [0,60000]; 0=never. Default 20000".into(),
input_schema: json!({
@@ -3893,9 +4052,10 @@ impl Tool for SetAgentCursorMotionTool {
"arc_size":{"type":"number","description":"Arc deflection as fraction of path length [0,1]. Default 0.25."},
"arc_flow":{"type":"number","description":"Asymmetry bias [-1,1]. Default 0.0."},
"spring":{"type":"number","description":"Settle damping [0.3,1.0]. Default 0.72."},
- "glide_duration_ms":{"type":"number","minimum":50,"maximum":5000,"description":"Flight duration per move in ms. Default 160."},
+ "glide_duration_ms":{"type":"number","minimum":50,"maximum":5000,"description":"Fixed flight duration per move in ms; omit for speed-based timing (the default)."},
"dwell_after_click_ms":{"type":"number","minimum":0,"maximum":5000,"description":"Pause after click ripple in ms. Default 80."},
- "idle_hide_ms":{"type":"number","minimum":0,"maximum":60000,"description":"Auto-hide delay in ms. 0=never. Default 20000."}
+ "idle_hide_ms":{"type":"number","minimum":0,"maximum":60000,"description":"Auto-hide delay in ms. 0=never. Default 20000."},
+ "turn_radius":{"type":"number","minimum":1,"maximum":1000,"description":"Minimum turning radius of the glide path in points; smaller = tighter curves. Default 80."}
},"additionalProperties":false
}),
read_only: false, destructive: false, idempotent: true, open_world: false,
@@ -3931,18 +4091,20 @@ impl Tool for SetAgentCursorMotionTool {
num(args.get("dwell_after_click_ms")),
num(args.get("idle_hide_ms")),
None, // press_duration_ms β€” not in Swift tool surface
+ num(args.get("turn_radius")),
);
crate::overlay::send_command(cursor_id.clone(), cursor_overlay::OverlayCommand::SetMotion(updated.clone()));
// Match Swift text format 1:1.
let summary = format!(
"cursor motion: startHandle={sh} endHandle={eh} arcSize={asz} arcFlow={af} \
- spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih}",
+ spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih} turnRadius={tr}",
sh = updated.start_handle, eh = updated.end_handle,
asz = updated.arc_size, af = updated.arc_flow,
sp = updated.spring,
gd = updated.glide_duration_ms as i64,
dw = updated.dwell_after_click_ms as i64,
ih = updated.idle_hide_ms as i64,
+ tr = updated.turn_radius as i64,
);
ToolResult::text(format!("βœ… {summary}")).with_structured(json!({
"cursor_id": cursor_id,
@@ -3954,6 +4116,7 @@ impl Tool for SetAgentCursorMotionTool {
"glide_duration_ms": updated.glide_duration_ms,
"dwell_after_click_ms": updated.dwell_after_click_ms,
"idle_hide_ms": updated.idle_hide_ms,
+ "turn_radius": updated.turn_radius,
}))
}
}
@@ -3989,13 +4152,14 @@ impl Tool for GetAgentCursorStateTool {
// Swift text format 1:1: single-line camelCase key=value pairs.
let summary = format!(
"cursor: enabled={enabled} startHandle={sh} endHandle={eh} arcSize={asz} \
- arcFlow={af} spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih}",
+ arcFlow={af} spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih} turnRadius={tr}",
sh = motion.start_handle, eh = motion.end_handle,
asz = motion.arc_size, af = motion.arc_flow,
sp = motion.spring,
gd = motion.glide_duration_ms as i64,
dw = motion.dwell_after_click_ms as i64,
ih = motion.idle_hide_ms as i64,
+ tr = motion.turn_radius as i64,
);
// Rust-only structured payload: the same fields + the multi-cursor
// instance map. Cursor instances are a Rust extension Swift doesn't
@@ -4012,6 +4176,7 @@ impl Tool for GetAgentCursorStateTool {
"glide_duration_ms": motion.glide_duration_ms,
"dwell_after_click_ms": motion.dwell_after_click_ms,
"idle_hide_ms": motion.idle_hide_ms,
+ "turn_radius": motion.turn_radius,
"cursors": cursors,
}))
}