Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(windows): background input without z-raise + multi-cursor demos + overlay improvements (#1809)
ddupont53bb84cc71cb
diff --git a/.gitignore b/.gitignoreindex 32a250ed36..a91bb24b5e 100644--- a/.gitignore+++ b/.gitignore@@ -217,4 +217,6 @@ post-provisionscripts/check-repo-md-links.pydocs/scripts/check-links.pydocs/scripts/check-all-links.py-docs/scripts/check-mdx-links.py\ No newline at end of file+docs/scripts/check-mdx-links.py+# Local Windows RE scratch (downloaded PDBs, disassembly scripts)+.re-windows/diff --git a/.vscode/settings.json b/.vscode/settings.jsonindex c06f93aadf..c2db3d0c77 100644--- a/.vscode/settings.json+++ b/.vscode/settings.json@@ -21,5 +21,18 @@],"mypy-type-checker.path": ["${workspaceFolder}"- ]+ ],+ "workbench.colorCustomizations": {+ "activityBar.background": "#410A56",+ "titleBar.activeBackground": "#5A0E78",+ "titleBar.activeForeground": "#FEFBFF",+ "titleBar.inactiveBackground": "#410A56",+ "titleBar.inactiveForeground": "#FEFBFF",+ "statusBar.background": "#410A56",+ "statusBar.foreground": "#FEFBFF",+ "statusBar.debuggingBackground": "#410A56",+ "statusBar.debuggingForeground": "#FEFBFF",+ "statusBar.noFolderBackground": "#410A56",+ "statusBar.noFolderForeground": "#FEFBFF"+ }}\ No newline at end of filediff --git a/demo/jukebox/Cargo.lock b/demo/jukebox/Cargo.locknew file mode 100644index 0000000000..10a42677ce--- /dev/null+++ b/demo/jukebox/Cargo.lock@@ -0,0 +1,1072 @@+# This file is automatically @generated by Cargo.+# It is not intended for manual editing.+version = 4++[[package]]+name = "aho-corasick"+version = "1.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"+dependencies = [+ "memchr",+]++[[package]]+name = "alsa"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43"+dependencies = [+ "alsa-sys",+ "bitflags 2.11.1",+ "cfg-if",+ "libc",+]++[[package]]+name = "alsa-sys"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527"+dependencies = [+ "libc",+ "pkg-config",+]++[[package]]+name = "autocfg"+version = "1.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"++[[package]]+name = "bindgen"+version = "0.72.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895"+dependencies = [+ "bitflags 2.11.1",+ "cexpr",+ "clang-sys",+ "itertools",+ "proc-macro2",+ "quote",+ "regex",+ "rustc-hash",+ "shlex 1.3.0",+ "syn",+]++[[package]]+name = "bitflags"+version = "1.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"++[[package]]+name = "bitflags"+version = "2.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"++[[package]]+name = "bumpalo"+version = "3.20.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"++[[package]]+name = "bytes"+version = "1.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"++[[package]]+name = "cc"+version = "1.2.63"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"+dependencies = [+ "find-msvc-tools",+ "jobserver",+ "libc",+ "shlex 2.0.1",+]++[[package]]+name = "cesu8"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"++[[package]]+name = "cexpr"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"+dependencies = [+ "nom",+]++[[package]]+name = "cfg-if"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"++[[package]]+name = "clang-sys"+version = "1.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"+dependencies = [+ "glob",+ "libc",+ "libloading",+]++[[package]]+name = "combine"+version = "4.6.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"+dependencies = [+ "bytes",+ "memchr",+]++[[package]]+name = "core-foundation-sys"+version = "0.8.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"++[[package]]+name = "coreaudio-rs"+version = "0.11.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "321077172d79c662f64f5071a03120748d5bb652f5231570141be24cfcd2bace"+dependencies = [+ "bitflags 1.3.2",+ "core-foundation-sys",+ "coreaudio-sys",+]++[[package]]+name = "coreaudio-sys"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ceec7a6067e62d6f931a2baf6f3a751f4a892595bcec1461a3c94ef9949864b6"+dependencies = [+ "bindgen",+]++[[package]]+name = "cpal"+version = "0.15.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "873dab07c8f743075e57f524c583985fbaf745602acbe916a01539364369a779"+dependencies = [+ "alsa",+ "core-foundation-sys",+ "coreaudio-rs",+ "dasp_sample",+ "jni",+ "js-sys",+ "libc",+ "mach2",+ "ndk",+ "ndk-context",+ "oboe",+ "wasm-bindgen",+ "wasm-bindgen-futures",+ "web-sys",+ "windows 0.54.0",+]++[[package]]+name = "dasp_sample"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0c87e182de0887fd5361989c677c4e8f5000cd9491d6d563161a8f3a5519fc7f"++[[package]]+name = "either"+version = "1.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"++[[package]]+name = "equivalent"+version = "1.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"++[[package]]+name = "find-msvc-tools"+version = "0.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"++[[package]]+name = "futures-core"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"++[[package]]+name = "futures-task"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"++[[package]]+name = "futures-util"+version = "0.3.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"+dependencies = [+ "futures-core",+ "futures-task",+ "pin-project-lite",+ "slab",+]++[[package]]+name = "getrandom"+version = "0.3.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"+dependencies = [+ "cfg-if",+ "libc",+ "r-efi",+ "wasip2",+]++[[package]]+name = "glob"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"++[[package]]+name = "hashbrown"+version = "0.17.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"++[[package]]+name = "indexmap"+version = "2.14.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"+dependencies = [+ "equivalent",+ "hashbrown",+]++[[package]]+name = "itertools"+version = "0.13.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"+dependencies = [+ "either",+]++[[package]]+name = "jni"+version = "0.21.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"+dependencies = [+ "cesu8",+ "cfg-if",+ "combine",+ "jni-sys 0.3.1",+ "log",+ "thiserror",+ "walkdir",+ "windows-sys 0.45.0",+]++[[package]]+name = "jni-sys"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"+dependencies = [+ "jni-sys 0.4.1",+]++[[package]]+name = "jni-sys"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"+dependencies = [+ "jni-sys-macros",+]++[[package]]+name = "jni-sys-macros"+version = "0.4.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"+dependencies = [+ "quote",+ "syn",+]++[[package]]+name = "jobserver"+version = "0.1.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"+dependencies = [+ "getrandom",+ "libc",+]++[[package]]+name = "js-sys"+version = "0.3.99"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"+dependencies = [+ "cfg-if",+ "futures-util",+ "once_cell",+ "wasm-bindgen",+]++[[package]]+name = "jukebox-app"+version = "0.1.0"+dependencies = [+ "rodio",+ "windows 0.58.0",+]++[[package]]+name = "jukebox-orchestrator"+version = "0.1.0"+dependencies = [+ "midly",+ "windows 0.58.0",+]++[[package]]+name = "libc"+version = "0.2.186"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"++[[package]]+name = "libloading"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"+dependencies = [+ "cfg-if",+ "windows-link",+]++[[package]]+name = "log"+version = "0.4.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"++[[package]]+name = "mach2"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44"+dependencies = [+ "libc",+]++[[package]]+name = "memchr"+version = "2.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"++[[package]]+name = "midly"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "207d755f4cb882d20c4da58d707ca9130a0c9bc5061f657a4f299b8e36362b7a"++[[package]]+name = "minimal-lexical"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"++[[package]]+name = "ndk"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2076a31b7010b17a38c01907c45b945e8f11495ee4dd588309718901b1f7a5b7"+dependencies = [+ "bitflags 2.11.1",+ "jni-sys 0.3.1",+ "log",+ "ndk-sys",+ "num_enum",+ "thiserror",+]++[[package]]+name = "ndk-context"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b"++[[package]]+name = "ndk-sys"+version = "0.5.0+25.2.9519653"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691"+dependencies = [+ "jni-sys 0.3.1",+]++[[package]]+name = "nom"+version = "7.1.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"+dependencies = [+ "memchr",+ "minimal-lexical",+]++[[package]]+name = "num-derive"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "num-traits"+version = "0.2.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"+dependencies = [+ "autocfg",+]++[[package]]+name = "num_enum"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"+dependencies = [+ "num_enum_derive",+ "rustversion",+]++[[package]]+name = "num_enum_derive"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"+dependencies = [+ "proc-macro-crate",+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "oboe"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e8b61bebd49e5d43f5f8cc7ee2891c16e0f41ec7954d36bcb6c14c5e0de867fb"+dependencies = [+ "jni",+ "ndk",+ "ndk-context",+ "num-derive",+ "num-traits",+ "oboe-sys",+]++[[package]]+name = "oboe-sys"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c8bb09a4a2b1d668170cfe0a7d5bc103f8999fb316c98099b6a9939c9f2e79d"+dependencies = [+ "cc",+]++[[package]]+name = "once_cell"+version = "1.21.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"++[[package]]+name = "pin-project-lite"+version = "0.2.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"++[[package]]+name = "pkg-config"+version = "0.3.33"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"++[[package]]+name = "proc-macro-crate"+version = "3.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"+dependencies = [+ "toml_edit",+]++[[package]]+name = "proc-macro2"+version = "1.0.106"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"+dependencies = [+ "unicode-ident",+]++[[package]]+name = "quote"+version = "1.0.45"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"+dependencies = [+ "proc-macro2",+]++[[package]]+name = "r-efi"+version = "5.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"++[[package]]+name = "regex"+version = "1.12.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-automata",+ "regex-syntax",+]++[[package]]+name = "regex-automata"+version = "0.4.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"+dependencies = [+ "aho-corasick",+ "memchr",+ "regex-syntax",+]++[[package]]+name = "regex-syntax"+version = "0.8.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"++[[package]]+name = "rodio"+version = "0.17.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b1bb7b48ee48471f55da122c0044fcc7600cfcc85db88240b89cb832935e611"+dependencies = [+ "cpal",+]++[[package]]+name = "rustc-hash"+version = "2.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"++[[package]]+name = "rustversion"+version = "1.0.22"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"++[[package]]+name = "same-file"+version = "1.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"+dependencies = [+ "winapi-util",+]++[[package]]+name = "serde_core"+version = "1.0.228"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"+dependencies = [+ "serde_derive",+]++[[package]]+name = "serde_derive"+version = "1.0.228"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "shlex"+version = "1.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"++[[package]]+name = "shlex"+version = "2.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"++[[package]]+name = "slab"+version = "0.4.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"++[[package]]+name = "syn"+version = "2.0.117"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "thiserror"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"+dependencies = [+ "thiserror-impl",+]++[[package]]+name = "thiserror-impl"+version = "1.0.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "toml_datetime"+version = "1.1.1+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"+dependencies = [+ "serde_core",+]++[[package]]+name = "toml_edit"+version = "0.25.12+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7"+dependencies = [+ "indexmap",+ "toml_datetime",+ "toml_parser",+ "winnow",+]++[[package]]+name = "toml_parser"+version = "1.1.2+spec-1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"+dependencies = [+ "winnow",+]++[[package]]+name = "unicode-ident"+version = "1.0.24"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"++[[package]]+name = "walkdir"+version = "2.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"+dependencies = [+ "same-file",+ "winapi-util",+]++[[package]]+name = "wasip2"+version = "1.0.3+wasi-0.2.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"+dependencies = [+ "wit-bindgen",+]++[[package]]+name = "wasm-bindgen"+version = "0.2.122"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"+dependencies = [+ "cfg-if",+ "once_cell",+ "rustversion",+ "wasm-bindgen-macro",+ "wasm-bindgen-shared",+]++[[package]]+name = "wasm-bindgen-futures"+version = "0.4.72"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"+dependencies = [+ "js-sys",+ "wasm-bindgen",+]++[[package]]+name = "wasm-bindgen-macro"+version = "0.2.122"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"+dependencies = [+ "quote",+ "wasm-bindgen-macro-support",+]++[[package]]+name = "wasm-bindgen-macro-support"+version = "0.2.122"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"+dependencies = [+ "bumpalo",+ "proc-macro2",+ "quote",+ "syn",+ "wasm-bindgen-shared",+]++[[package]]+name = "wasm-bindgen-shared"+version = "0.2.122"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"+dependencies = [+ "unicode-ident",+]++[[package]]+name = "web-sys"+version = "0.3.99"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"+dependencies = [+ "js-sys",+ "wasm-bindgen",+]++[[package]]+name = "winapi-util"+version = "0.1.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"+dependencies = [+ "windows-sys 0.61.2",+]++[[package]]+name = "windows"+version = "0.54.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49"+dependencies = [+ "windows-core 0.54.0",+ "windows-targets 0.52.6",+]++[[package]]+name = "windows"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"+dependencies = [+ "windows-core 0.58.0",+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-core"+version = "0.54.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65"+dependencies = [+ "windows-result 0.1.2",+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-core"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"+dependencies = [+ "windows-implement",+ "windows-interface",+ "windows-result 0.2.0",+ "windows-strings",+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-implement"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-interface"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-link"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"++[[package]]+name = "windows-result"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"+dependencies = [+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-result"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"+dependencies = [+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-strings"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"+dependencies = [+ "windows-result 0.2.0",+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-sys"+version = "0.45.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"+dependencies = [+ "windows-targets 0.42.2",+]++[[package]]+name = "windows-sys"+version = "0.61.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"+dependencies = [+ "windows-link",+]++[[package]]+name = "windows-targets"+version = "0.42.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"+dependencies = [+ "windows_aarch64_gnullvm 0.42.2",+ "windows_aarch64_msvc 0.42.2",+ "windows_i686_gnu 0.42.2",+ "windows_i686_msvc 0.42.2",+ "windows_x86_64_gnu 0.42.2",+ "windows_x86_64_gnullvm 0.42.2",+ "windows_x86_64_msvc 0.42.2",+]++[[package]]+name = "windows-targets"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"+dependencies = [+ "windows_aarch64_gnullvm 0.52.6",+ "windows_aarch64_msvc 0.52.6",+ "windows_i686_gnu 0.52.6",+ "windows_i686_gnullvm",+ "windows_i686_msvc 0.52.6",+ "windows_x86_64_gnu 0.52.6",+ "windows_x86_64_gnullvm 0.52.6",+ "windows_x86_64_msvc 0.52.6",+]++[[package]]+name = "windows_aarch64_gnullvm"+version = "0.42.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"++[[package]]+name = "windows_aarch64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"++[[package]]+name = "windows_aarch64_msvc"+version = "0.42.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"++[[package]]+name = "windows_aarch64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"++[[package]]+name = "windows_i686_gnu"+version = "0.42.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"++[[package]]+name = "windows_i686_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"++[[package]]+name = "windows_i686_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"++[[package]]+name = "windows_i686_msvc"+version = "0.42.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"++[[package]]+name = "windows_i686_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"++[[package]]+name = "windows_x86_64_gnu"+version = "0.42.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"++[[package]]+name = "windows_x86_64_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"++[[package]]+name = "windows_x86_64_gnullvm"+version = "0.42.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"++[[package]]+name = "windows_x86_64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"++[[package]]+name = "windows_x86_64_msvc"+version = "0.42.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"++[[package]]+name = "windows_x86_64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"++[[package]]+name = "winnow"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"+dependencies = [+ "memchr",+]++[[package]]+name = "wit-bindgen"+version = "0.57.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"diff --git a/demo/jukebox/Cargo.toml b/demo/jukebox/Cargo.tomlnew file mode 100644index 0000000000..9408253c7e--- /dev/null+++ b/demo/jukebox/Cargo.toml@@ -0,0 +1,6 @@+[workspace]+resolver = "2"+members = ["app", "orchestrator"]++[profile.release]+opt-level = 2diff --git a/demo/jukebox/README.md b/demo/jukebox/README.mdnew file mode 100644index 0000000000..1d29461738--- /dev/null+++ b/demo/jukebox/README.md@@ -0,0 +1,190 @@+# CUA Jukebox β coordinated multi-cursor computer-use, as music++A MIDI file (or a built-in demo song) becomes a **grid of miniwob-style+minigame windows** β one per track. Each window gets its **own cua-driver+session = its own uniquely-coloured agent cursor**. While the song plays, every+note steers that track's cursor onto its widget and **clicks it in the+background** β and the click is what makes the sound. One cursor per part, one+colour per agent, all driven off a single clock: the dumbest possible orchestra,+performed entirely by background computer-use.++It's the native-windows sibling of the multi-cursor "National Records System"+demo: same no-z-raise, no-real-mouse-movement background actuation, but here the+*timing* is the point.++```+βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+β CUA JUKEBOX β Transport βΆ PLAY βββββββββββββββ β β you click PLAY+ββββββββββββββββ¬βββββββββββββββ¬βββββββββββββββ¬βββββββββββββββ¬βββββ€+β Bass β£crimsonβ Kick β£amber β Hat β£aqua β Pad β£mint β β¦ β+β [pitch strip] β [ KICK pad ] β [ HAT pad ] β [pitch strip]β β β one agent+β β β β β β cursor each+ββββββββββββββββ΄βββββββββββββββ΄βββββββββββββββ΄βββββββββββββββ΄βββββ+```++## What each window is++| Window | Role | Widget | cursor actuation |+|---|---|---|---|+| Transport (controller) | foreground; the one human action | real Win32 `PLAY/STOP` button + playhead | you click it |+| Kick / Snare / Hat | drum pad | one big pad; any click = a fixed hit | background click β drum voice |+| Bass / Lead / Pad / Arp / β¦ | melodic | a **pitch strip**: the click's X selects the semitone | background click β that pitch |++Each instrument owns its **own `rodio` output stream**, so notes from the+separate instrument processes mix at the OS mixer β genuine polyphony across the+whole fleet. The track's name picks its minigame + synth voice (kick/snare/hat β+pad; bass/lead/pad/arp/β¦ β pitch strip), mirroring the original HTML jukebox's+`inferKind`.++**Visual feedback:** a melodic strip lights the struck **key**, and each press+fades on its own clock β so a chord lights several keys at once, individually. A+drum pad instead has a single **brightness that gets an impulse per hit and+constantly fades**, so the faster it's triggered the brighter it glows.++## Cursor pools (one colour, many hands)++A track normally has one cursor, but a cursor is "busy" for the whole click β+the glide **plus** dispatch (β glide + 130 ms). When a track's notes fall closer+together than that β a **chord** (simultaneous), or just a line faster than one+cursor can service at the current glide β the first cursor is still busy, so the+next free cursor in the track's **pool** takes the note, and the pool grows on+demand (try the first, else the next, else spawn one; capped at 6). Every pool+member is forced to the **same colour**, so a triad fans out into three+identically-coloured cursors stabbing three keys at once, and a fast hi-hat line+splits across two. Sizing the pool to the *real* click duration (not just the+glide) is what keeps every cursor on the beat β otherwise one cursor would fall+progressively behind on a track whose notes outpace its glide. The built-in demo+plays Pad triads and dense hats/arps to show this. Slow / sparse tracks keep one+cursor. Each pool cursor is its own cua-driver session + its own persistent+connection + its own thread, so they actuate concurrently.++## How the timing stays on the beat++A background click only makes its sound once the cursor has *glided onto* the+widget and tapped it, so the actuation lags the dispatch. Three things make it+land on the beat anyway:++1. **Fixed-duration glide.** Every cursor is pinned to a known, constant flight+ time via `set_agent_cursor_motion {"glide_duration_ms": 200}` β so a 3-pixel+ nudge and a cross-strip leap both arrive in the same 200 ms, making the+ latency *predictable enough to sequence*.++ > `glide_duration_ms` is honoured identically on macOS, Windows, and Linux β+ > it lives in the shared cursor-overlay render core (`tick_motion` /+ > `tick_swift_constants`). `0` (the default) keeps the original speed-based+ > glide; any value `50β5000` forces that fixed flight time. No platform drift.++2. **Persistent daemon connection.** The orchestrator holds **one named-pipe+ connection per voice** to `cua-driver serve` and pipelines every click over+ it β no `cua-driver call` *process spawn per note*. That spawn (tens of ms,+ wildly variable) was the original timing-jitter source; removing it dropped+ the per-note jitter from **sd β 490 ms β β 35 ms**.++3. **Per-voice adaptive lead + throughput-sized pools + 1 ms timer.** Each click+ is fired early by an adaptive lead (a per-voice EMA tuned from the measured+ error, warm-started at the dispatch overhead) so the mean error β 0 without+ one congested track skewing another. Pools are sized to the real click+ duration (see above) so no single cursor outruns its glide. The system timer+ is raised to 1 ms so `thread::sleep` schedules each click precisely. **And the+ daemon must be built `--release`** β the overlay's per-frame pixel pipeline is+ ~5Γ slower in debug (β12 fps vs β60 fps under the full demo), and a slow+ overlay = late, jittery glide-arrivals.++The orchestrator **tracks the diff** itself: it records `(actual β scheduled)`+for every note and prints a report at song end, e.g.++```+[timing] n=256 mean=-4.0ms |mean|=19.4ms sd=35.8ms median=-0.8ms p10=-36 p90=+20 max=138+```++i.e. notes land on the beat to a sub-millisecond median with ~35 ms of jitter+(about as tight as a human drummer), with a 200 ms glide. Tune with env vars:+`JUKEBOX_GLIDE_MS` (default 200) and `JUKEBOX_LEAD_MS` (defaults to the glide,+the initial per-voice pre-roll the adaptive correction refines from).++## Build++```powershell+# from this directory+cargo build+# Build the driver RELEASE β the cursor overlay composites a full-virtual-screen+# bitmap (RGBAβBGRA) and blits it every frame, which is ~5Γ slower unoptimized.+# Debug: ~12 fps overlay under the full demo; release: ~60 fps (vsync-capped).+cargo build -p cua-driver --release --manifest-path ..\..\libs\cua-driver\rust\Cargo.toml+$env:CUA_DRIVER_EXE = "..\..\libs\cua-driver\rust\target\release\cua-driver.exe"+```++## Run++```powershell+.\target\debug\jukebox-orchestrator.exe # then click βΆ PLAY in the Transport window+.\target\debug\jukebox-orchestrator.exe --auto # self-starts after warmup+.\target\debug\jukebox-orchestrator.exe song.mid # drive any multitrack .mid (best with named tracks)+.\target\debug\jukebox-orchestrator.exe song.mid --auto+```++The orchestrator reaps any stale daemon, starts `cua-driver serve`, launches the+Transport + one instrument window per track, tiles them (a 600Γ80 bar over a grid+of 200Γ160 tiles), arms one coloured fixed-glide cursor pool per track, and on+**PLAY** fans beat-synced background clicks out to every instrument. Press **Esc**+on the Transport (or Ctrl-C the orchestrator) to tear everything down β a Windows+Job Object kills the whole tree.++## Bring your own song (MIDI)++- **Drag a `.mid` onto `jukebox-orchestrator.exe`** in Explorer β Windows passes+ it as the argument, so the file becomes the song.+- β¦or pass it on the command line: `jukebox-orchestrator.exe path\to\song.mid`.++It works best with **multitrack files that have named tracks** β the visualizer+reads each track's name to pick its minigame + voice (`kick`/`snare`/`hat` β drum+pad; `bass`/`lead`/`pad`/`arp`/`string`/β¦ β pitch strip), and an unnamed track+falls back to a sine pitch-strip. General-MIDI pop/electronic arrangements (one+instrument per track, a drum track, a bass, a couple of leads/pads) map cleanly;+chords on a track fan out into that track's same-colour cursor pool.++Where to find MIDIs (always check each file's own licence):+- **Open / Creative-Commons** score libraries β [Mutopia Project](https://www.mutopiaproject.org/),+ [kunstderfuge](https://creativecommons.org/2008/03/07/kunstderfuge/) (CC BY-NC-SA),+ the [Classical Piano MIDI Page](http://piano-midi.de/copy.htm) (CC BY-SA) β these+ skew classical but are cleanly licensed and well-separated into tracks.+- **CC audio search**: [Openverse](https://openverse.org/) and the+ [Free Music Archive](https://freemusicarchive.org/curator/Creative_Commons/).+- **Large general archives** (free downloads; licensing varies per file, so use for+ personal/demo use): BitMidi (`bitmidi.com`), MidiWorld (`midiworld.com`),+ FreeMidi (`freemidi.org`) β good for finding multitrack electronic/pop tracks.++No `.mid`? The built-in generated 8-bar electronic loop (the default) is tuned to+exercise every part of the visualizer β drums, a bass line, an arp, and Pad+triads that show the cursor pool.++### Env overrides+`CUA_DRIVER_EXE`, `JUKEBOX_APP_EXE`, `JUKEBOX_GLIDE_MS` (default 200),+`JUKEBOX_LEAD_MS` (defaults to the glide). Set+`CUA_DRIVER_RS_OVERLAY_FPS_FILE=<path>` (read by the daemon) to log the agent+cursor overlay's measured render FPS once a second.++## How the coloured cursors work++Each track's session is a **cua-driver palette name** (`crimson`, `amber`,+`aqua`, `mint_lime`, `orchid`, β¦), so its overlay cursor renders in that palette+automatically (`Palette::for_instance(session)`) β the same trick the+multi-cursor demo uses. The instrument window's accent and the Transport legend+reuse that palette's colour, so the cursor, its window, and the legend all read+as one colour. (`cursor_color` on `set_agent_cursor_motion` only records a value;+it doesn't repaint the overlay β true on macOS and Windows alike β so we key by+palette name instead.)++## Honest caveats++- **Timing is groove-tight (median ~0 ms, jitter ~30 ms), not sample-accurate.**+ The residual jitter is the overlay render tick (~8 ms) + OS scheduling. Under+ heavy system load the daemon's single overlay render thread can starve, which+ shows up as occasional multi-hundred-ms outliers on dense tracks; on an idle+ machine all notes land in the Β±30 ms band.+- **MIDI parsing uses a single tempo** (first tempo event wins) and ignores+ channel/program data β instrument inference leans on track names. Untitled+ tracks default to a sine pitch-strip.+- Up to **9 tracks** (one per cua-driver palette); extra tracks are dropped.+- Audio needs a default output device; with none, instruments still flash+ silently.diff --git a/demo/jukebox/app/Cargo.toml b/demo/jukebox/app/Cargo.tomlnew file mode 100644index 0000000000..5e92d2721f--- /dev/null+++ b/demo/jukebox/app/Cargo.toml@@ -0,0 +1,20 @@+[package]+name = "jukebox-app"+version = "0.1.0"+edition = "2021"++[[bin]]+name = "jukebox-app"+path = "src/main.rs"++[dependencies]+# Each instrument window owns its own audio output, so concurrent notes from+# separate processes mix at the OS mixer β genuine multi-process polyphony.+rodio = { version = "0.17", default-features = false }+windows = { version = "0.58", features = [+ "Win32_Foundation",+ "Win32_UI_WindowsAndMessaging",+ "Win32_UI_Shell",+ "Win32_Graphics_Gdi",+ "Win32_System_LibraryLoader",+] }diff --git a/demo/jukebox/app/src/main.rs b/demo/jukebox/app/src/main.rsnew file mode 100644index 0000000000..c3d149d6d5--- /dev/null+++ b/demo/jukebox/app/src/main.rs@@ -0,0 +1,604 @@+//! "CUA JUKEBOX" β one window per part of a song, each a tiny miniwob-style+//! minigame that an agent cursor actuates in time with the music. This binary+//! is BOTH window kinds the demo launches:+//!+//! controller β the transport: a big PLAY/STOP button (a real Win32 control+//! the human clicks), a track list, and a playhead. Emits+//! `PLAY` / `STOP` on stdout for the orchestrator to react to.+//!+//! instrument β one minigame + its own synth voice. When something clicks+//! the widget (the orchestrator drives these via cua-driver in+//! the background), the window plays its note and flashes. Two+//! widget kinds:+//! pad β a single drum pad; any click = a fixed hit.+//! keys β a pitch strip; the click's X selects the semitone,+//! so the orchestrator "plays a melody" by choosing+//! where on the strip each note lands.+//!+//! Every instrument owns its own `rodio` output stream, so notes from the+//! separate instrument processes mix at the OS audio mixer β real polyphony+//! across the whole coordinated fleet.++#![windows_subsystem = "windows"]++use std::cell::RefCell;+use std::f32::consts::PI;+use std::io::Write;+use std::time::Instant;++use windows::core::{w, PCWSTR};+use windows::Win32::Foundation::{COLORREF, HINSTANCE, HWND, LPARAM, LRESULT, RECT, WPARAM};+use windows::Win32::Graphics::Gdi::{+ BeginPaint, BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, CreateFontW, CreateSolidBrush,+ DeleteDC, DeleteObject, DrawTextW, EndPaint, FillRect, FrameRect, InvalidateRect, SelectObject,+ SetBkMode, SetTextColor, DT_CENTER, DT_LEFT, DT_SINGLELINE, DT_VCENTER, HBRUSH, HFONT,+ PAINTSTRUCT, SRCCOPY, TRANSPARENT,+};+use windows::Win32::System::LibraryLoader::GetModuleHandleW;+use windows::Win32::UI::Shell::{DragAcceptFiles, DragFinish, DragQueryFileW, HDROP};+use windows::Win32::UI::WindowsAndMessaging::*;++// ββ widget geometry (client fractions) β MUST match orchestrator's targets ββββ+const WX0: f64 = 0.06; // widget left+const WX1: f64 = 0.94; // widget right+const WY0: f64 = 0.34; // widget top+const WY1: f64 = 0.92; // widget bottom+const KEYS_SPAN: i32 = 24; // semitones across a `keys` strip (2 octaves)++// ββ synth βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+#[derive(Clone, Copy, PartialEq)]+enum Wave { Sine, Square, Saw, Triangle, Kick, Snare, Hat }++impl Wave {+ fn parse(s: &str) -> Wave {+ match s {+ "sine" => Wave::Sine,+ "square" => Wave::Square,+ "saw" => Wave::Saw,+ "triangle" => Wave::Triangle,+ "kick" => Wave::Kick,+ "snare" => Wave::Snare,+ "hat" => Wave::Hat,+ _ => Wave::Sine,+ }+ }+}++fn midi_to_freq(m: f32) -> f32 { 440.0 * 2f32.powf((m - 69.0) / 12.0) }++/// A short enveloped oscillator. One per note; rodio mixes overlapping ones.+struct Tone {+ sr: u32,+ idx: u32,+ total: u32,+ phase: f32,+ wave: Wave,+ amp: f32,+ f0: f32, // start frequency+ f1: f32, // end frequency (0 = no pitch sweep)+ rng: u32, // xorshift noise state+ prev_noise: f32,+}++impl Tone {+ fn note(wave: Wave, freq: f32, vel: f32) -> Tone {+ let sr = 44_100u32;+ let v = (vel / 127.0).clamp(0.05, 1.0);+ let (dur, amp, f0, f1) = match wave {+ Wave::Kick => (0.20, 0.55 * v, 155.0, 48.0),+ Wave::Snare => (0.18, 0.34 * v, 180.0, 0.0),+ Wave::Hat => (0.05, 0.24 * v, 9000.0, 0.0),+ _ => (0.34, 0.17 * v, freq, 0.0),+ };+ Tone {+ sr,+ idx: 0,+ total: (dur * sr as f32) as u32,+ phase: 0.0,+ wave,+ amp,+ f0,+ f1,+ rng: 0x9E37_79B9 ^ (freq as u32).wrapping_mul(2654435761),+ prev_noise: 0.0,+ }+ }+ fn noise(&mut self) -> f32 {+ // xorshift32+ let mut x = self.rng;+ x ^= x << 13;+ x ^= x >> 17;+ x ^= x << 5;+ self.rng = x;+ (x as f32 / u32::MAX as f32) * 2.0 - 1.0+ }+}++impl Iterator for Tone {+ type Item = f32;+ fn next(&mut self) -> Option<f32> {+ if self.idx >= self.total { return None; }+ let dur = self.total as f32 / self.sr as f32;+ let t = self.idx as f32 / self.sr as f32;+ let p = self.idx as f32 / self.total as f32;+ let atk = 0.004;+ let env = if t < atk { t / atk } else { (1.0 - (t - atk) / (dur - atk)).max(0.0).powf(1.6) };+ let freq = if self.f1 > 0.0 { self.f0 * (self.f1 / self.f0).powf(p) } else { self.f0 };+ self.phase += 2.0 * PI * freq / self.sr as f32;+ if self.phase > 2.0 * PI { self.phase -= 2.0 * PI; }+ let osc = match self.wave {+ Wave::Sine | Wave::Kick => self.phase.sin(),+ Wave::Square => if self.phase.sin() >= 0.0 { 1.0 } else { -1.0 },+ Wave::Saw => self.phase / PI - 1.0,+ Wave::Triangle => (2.0 / PI) * self.phase.sin().asin(),+ Wave::Snare => { let n = self.noise(); 0.7 * n + 0.3 * self.phase.sin() }+ Wave::Hat => { let n = self.noise(); let hp = n - self.prev_noise; self.prev_noise = n; hp }+ };+ self.idx += 1;+ Some(osc * env * self.amp)+ }+}++impl rodio::Source for Tone {+ fn current_frame_len(&self) -> Option<usize> { None }+ fn channels(&self) -> u16 { 1 }+ fn sample_rate(&self) -> u32 { self.sr }+ fn total_duration(&self) -> Option<std::time::Duration> { None }+}++// ββ state ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+#[derive(Clone, Copy, PartialEq)]+enum Mode { Controller, Instrument }++#[derive(Clone, Copy, PartialEq)]+enum Kind { Pad, Keys, Drums }++/// One note actuation's fading highlight. `key` is the strip cell it lit (-1 for+/// a drum pad); `inten` fades 1β0 and `age` grows so a pad press expands an+/// outward ring. Each pulse fades on its own clock, so overlapping presses stay+/// individually visible.+#[derive(Clone, Copy)]+struct Pulse { key: i32, inten: f32, age: f32 }++struct Audio {+ _stream: rodio::OutputStream,+ handle: rodio::OutputStreamHandle,+}++struct State {+ mode: Mode,+ cw: i32,+ ch: i32,+ accent: COLORREF,+ accent_rgb: (u8, u8, u8),+ label: String,+ f_title: HFONT,+ f_label: HFONT,+ f_big: HFONT,++ // instrument+ kind: Kind,+ wave: Wave,+ root: f32,+ /// Semitones the pitch strip spans (fit to the track's range by the+ /// orchestrator) β keys mapping + cell count use this, not the constant.+ span: i32,+ /// Melodic strips: recent note actuations, each fading on its own clock so+ /// a chord fanned across the pool lights several keys at once, individually.+ pulses: Vec<Pulse>,+ /// Drum pads: a single brightness that constantly fades and gets an impulse+ /// on each hit β so the faster you trigger it, the brighter it glows.+ pad_glow: f32,+ /// Drum kits: one such brightness per kick/snare/hat zone.+ zone_glow: [f32; 3],+ hits: u32,+ audio: Option<Audio>,++ // controller+ bpm: u32,+ dur_ms: u64,+ tracks: Vec<(String, COLORREF)>,+ hbtn: HWND,+ playing: bool,+ play_start: Option<Instant>,+ /// Song time (ms) already elapsed in finished segments β frozen while paused+ /// so the playhead (and the orchestrator's resume offset) hold position.+ accum_ms: u64,+}++thread_local! { static STATE: RefCell<Option<State>> = const { RefCell::new(None) }; }++const ID_PLAY: isize = 2001;+// Single transport button is an icon: βΆ when stopped/paused, βΈ while playing.+const ICON_PLAY: PCWSTR = w!("βΆ");+const ICON_PAUSE: PCWSTR = w!("βΈ");++fn emit(line: &str) { let _ = writeln!(std::io::stdout(), "{line}"); let _ = std::io::stdout().flush(); }+fn wide(s: &str) -> Vec<u16> { s.encode_utf16().chain(std::iter::once(0)).collect() }+fn arg(args: &[String], flag: &str) -> Option<String> {+ args.iter().position(|a| a == flag).and_then(|i| args.get(i + 1).cloned())+}+fn parse_color(s: &str) -> (COLORREF, (u8, u8, u8)) {+ let h = s.trim_start_matches('#');+ let v = u32::from_str_radix(h, 16).unwrap_or(0x00FFFF);+ let (r, g, b) = (((v >> 16) & 255) as u8, ((v >> 8) & 255) as u8, (v & 255) as u8);+ (COLORREF(((b as u32) << 16) | ((g as u32) << 8) | r as u32), (r, g, b))+}+fn rgb(r: u8, g: u8, b: u8) -> COLORREF { COLORREF(((b as u32) << 16) | ((g as u32) << 8) | r as u32) }+fn mix(a: (u8, u8, u8), b: (u8, u8, u8), t: f32) -> COLORREF {+ let f = |x: u8, y: u8| (x as f32 + (y as f32 - x as f32) * t) as u8;+ rgb(f(a.0, b.0), f(a.1, b.1), f(a.2, b.2))+}+fn fr(cw: i32, ch: i32, l: f64, t: f64, r: f64, b: f64) -> RECT {+ RECT { left: (cw as f64 * l) as i32, top: (ch as f64 * t) as i32,+ right: (cw as f64 * r) as i32, bottom: (ch as f64 * b) as i32 }+}+unsafe fn mk_font(h: i32, w: i32, face: PCWSTR) -> HFONT { CreateFontW(h, 0, 0, 0, w, 0, 0, 0, 0, 0, 0, 0, 0, face) }++fn main() {+ let args: Vec<String> = std::env::args().collect();+ let mode = if args.get(1).map(|s| s.as_str()) == Some("controller") { Mode::Controller } else { Mode::Instrument };+ let title = arg(&args, "--title").unwrap_or_else(|| "JUKEBOX".into());+ let (accent, accent_rgb) = parse_color(&arg(&args, "--color").unwrap_or_else(|| "#3bd0ff".into()));+ let label = arg(&args, "--label").unwrap_or_else(|| "PART".into());+ let kind = match arg(&args, "--kind").as_deref() {+ Some("keys") => Kind::Keys,+ Some("drums") => Kind::Drums,+ _ => Kind::Pad,+ };+ let wave = Wave::parse(&arg(&args, "--wave").unwrap_or_else(|| "sine".into()));+ let root: f32 = arg(&args, "--root").and_then(|s| s.parse().ok()).unwrap_or(48.0);+ let span: i32 = arg(&args, "--span").and_then(|s| s.parse().ok()).unwrap_or(KEYS_SPAN).clamp(1, 96);+ let bpm: u32 = arg(&args, "--bpm").and_then(|s| s.parse().ok()).unwrap_or(120);+ let dur_ms: u64 = arg(&args, "--dur-ms").and_then(|s| s.parse().ok()).unwrap_or(16000);+ let tracks: Vec<(String, COLORREF)> = arg(&args, "--tracks").map(|s| s.split(',')+ .filter_map(|t| { let mut it = t.splitn(2, '|'); Some((it.next()?.to_string(), parse_color(it.next().unwrap_or("#888")).0)) })+ .collect()).unwrap_or_default();++ let audio = if mode == Mode::Instrument {+ rodio::OutputStream::try_default().ok().map(|(s, h)| Audio { _stream: s, handle: h })+ } else { None };++ unsafe {+ let hmod = GetModuleHandleW(None).unwrap();+ let class = w!("CuaJukeboxWindow");+ let bg = CreateSolidBrush(rgb(0x0b, 0x0c, 0x12));+ let wc = WNDCLASSW { lpfnWndProc: Some(wnd_proc), hInstance: hmod.into(), lpszClassName: class,+ hbrBackground: bg, hCursor: LoadCursorW(None, IDC_ARROW).unwrap_or_default(), ..Default::default() };+ RegisterClassW(&wc);++ STATE.with(|s| *s.borrow_mut() = Some(State {+ mode, cw: 0, ch: 0, accent, accent_rgb, label,+ f_title: HFONT::default(), f_label: HFONT::default(), f_big: HFONT::default(),+ kind, wave, root, span, pulses: Vec::new(), pad_glow: 0.0, zone_glow: [0.0; 3], hits: 0, audio,+ bpm, dur_ms, tracks, hbtn: HWND::default(),+ playing: false, play_start: None, accum_ms: 0,+ }));++ let tw = wide(&title);+ // Fixed sizes the orchestrator also lays out against: a thin 600Γ80+ // transport bar, and tight 200Γ160 instrument tiles. Borderless+ // (WS_POPUP) so the tiles pack together with no caption/frame β the+ // whole client is the visualizer. (Esc on the focused transport quits.)+ let (dw, dh) = if mode == Mode::Controller { (600, 80) } else { (200, 160) };+ let hwnd = CreateWindowExW(WINDOW_EX_STYLE(0), class, PCWSTR(tw.as_ptr()), WS_POPUP | WS_VISIBLE,+ CW_USEDEFAULT, CW_USEDEFAULT, dw, dh, None, None, HINSTANCE(hmod.0), None).expect("CreateWindowExW");+ let _ = ShowWindow(hwnd, SW_SHOWNORMAL);+ SetTimer(hwnd, 1, 40, None); // ~25fps: glow fade / playhead (region-clipped)++ let mut msg = MSG::default();+ while GetMessageW(&mut msg, None, 0, 0).as_bool() { let _ = TranslateMessage(&msg); DispatchMessageW(&msg); }+ }+}++unsafe fn relayout(hwnd: HWND, cw: i32, ch: i32) {+ STATE.with(|s| {+ let mut b = s.borrow_mut(); let Some(st) = b.as_mut() else { return };+ st.cw = cw; st.ch = ch;+ for f in [st.f_title, st.f_label, st.f_big] { if !f.is_invalid() { let _ = DeleteObject(f); } }+ let seg = wide("Segoe UI"); let mono = wide("Consolas");+ st.f_title = mk_font(-(ch / 16).clamp(14, 30), 800, PCWSTR(seg.as_ptr()));+ st.f_label = mk_font(-(ch / 26).clamp(11, 18), 500, PCWSTR(mono.as_ptr()));+ st.f_big = mk_font(-(ch / 9).clamp(20, 64), 800, PCWSTR(seg.as_ptr()));+ if st.mode == Mode::Controller && !st.hbtn.0.is_null() {+ // Thin transport bar: one small square icon button at the left edge,+ // leaving room for the title. Big font so the βΆ / βΈ glyph reads.+ let r = fr(cw, ch, 0.012, 0.16, 0.085, 0.84);+ let _ = MoveWindow(st.hbtn, r.left, r.top, r.right - r.left, r.bottom - r.top, true);+ SendMessageW(st.hbtn, WM_SETFONT, WPARAM(st.f_big.0 as usize), LPARAM(1));+ }+ });+ let _ = InvalidateRect(hwnd, None, true);+}++extern "system" fn wnd_proc(hwnd: HWND, msg: u32, wp: WPARAM, lp: LPARAM) -> LRESULT {+ unsafe {+ match msg {+ WM_CREATE => {+ if STATE.with(|s| s.borrow().as_ref().map(|st| st.mode)) == Some(Mode::Controller) {+ let hinst = HINSTANCE(GetModuleHandleW(None).unwrap().0);+ let hbtn = CreateWindowExW(+ WINDOW_EX_STYLE(0), w!("BUTTON"), ICON_PLAY,+ WS_CHILD | WS_VISIBLE | WINDOW_STYLE((BS_PUSHBUTTON | BS_CENTER | BS_VCENTER) as u32),+ 0, 0, 10, 10, hwnd, HMENU(ID_PLAY as *mut core::ffi::c_void), hinst, None).unwrap_or_default();+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() { st.hbtn = hbtn; });+ DragAcceptFiles(hwnd, true); // drop a .mid on the transport to load it+ }+ LRESULT(0)+ }+ WM_DROPFILES => {+ let hdrop = HDROP(wp.0 as *mut core::ffi::c_void);+ let mut buf = [0u16; 1024];+ let n = DragQueryFileW(hdrop, 0, Some(&mut buf));+ if n > 0 {+ let path = String::from_utf16_lossy(&buf[..n as usize]);+ emit(&format!("LOAD\t{path}")); // orchestrator restarts with this track+ }+ DragFinish(hdrop);+ LRESULT(0)+ }+ WM_SIZE => { let (cw, ch) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32); if cw > 0 && ch > 0 { relayout(hwnd, cw, ch); } LRESULT(0) }+ WM_COMMAND => {+ let (id, code) = ((wp.0 & 0xFFFF) as isize, ((wp.0 >> 16) & 0xFFFF) as u32);+ if code == BN_CLICKED && id == ID_PLAY { on_play_toggle(hwnd); }+ LRESULT(0)+ }+ WM_LBUTTONDOWN => {+ if STATE.with(|s| s.borrow().as_ref().map(|st| st.mode)) == Some(Mode::Instrument) {+ let x = (lp.0 & 0xFFFF) as i16 as i32;+ actuate(hwnd, x);+ }+ LRESULT(0)+ }+ WM_TIMER => { on_tick(hwnd); LRESULT(0) }+ // Esc quits (borderless windows have no close button); closing the+ // transport EOFs its stdout, which tears the whole demo down.+ WM_KEYDOWN if wp.0 == 0x1B => { PostQuitMessage(0); LRESULT(0) }+ // We fully repaint every frame via a double-buffered WM_PAINT, so+ // suppress the default background erase β that erase-then-paint is+ // the other half of GDI flicker.+ WM_ERASEBKGND => LRESULT(1),+ WM_PAINT => { paint(hwnd); LRESULT(0) }+ WM_DESTROY => { PostQuitMessage(0); LRESULT(0) }+ _ => DefWindowProcW(hwnd, msg, wp, lp),+ }+ }+}++/// Instrument actuation: the click's X selects what to play (pitch on a strip,+/// or which kick/snare/hat zone on a kit), then play the voice + flash. Fires+/// whether the click came from the human or β the point β from cua-driver.+unsafe fn actuate(hwnd: HWND, x: i32) {+ STATE.with(|s| {+ let mut b = s.borrow_mut(); let Some(st) = b.as_mut() else { return };+ let cw = st.cw.max(1) as f64;+ let frac = ((x as f64 / cw) - WX0) / (WX1 - WX0);+ let (key, freq, wave, vel) = match st.kind {+ Kind::Keys => {+ let key = (frac * st.span as f64).floor().clamp(0.0, (st.span - 1) as f64) as i32;+ // Each press is its OWN pulse, fading independently β a chord+ // lights several keys at once, each on its own clock.+ st.pulses.push(Pulse { key, inten: 1.0, age: 0.0 });+ if st.pulses.len() > 32 { st.pulses.remove(0); }+ (key, midi_to_freq(st.root + key as f32), st.wave, 104.0)+ }+ Kind::Drums => {+ // X selects the kick/snare/hat zone; light that zone's brightness.+ let z = (frac * 3.0).floor().clamp(0.0, 2.0) as usize;+ st.zone_glow[z] = (st.zone_glow[z] + 0.5).min(1.0);+ let w = [Wave::Kick, Wave::Snare, Wave::Hat][z];+ (z as i32, 0.0, w, 112.0)+ }+ Kind::Pad => {+ // Impulse on a constantly-fading brightness: rapid hits+ // accumulate toward full, sparse hits fade between them.+ st.pad_glow = (st.pad_glow + 0.5).min(1.0);+ (-1, midi_to_freq(st.root), st.wave, 112.0)+ }+ };+ st.hits += 1;+ if let Some(a) = &st.audio { let _ = a.handle.play_raw(Tone::note(wave, freq, vel)); }+ // Heartbeat for the orchestrator / verification (stdout is null in+ // normal runs, so this is a no-op there).+ emit(&format!("HIT {} key={} {:.0}Hz", st.hits, key, freq));+ });+ let _ = InvalidateRect(hwnd, None, false);+}++/// The single transport button is a play/pause toggle: playing β pause (freeze+/// at the current position so the next press resumes from here); paused/stopped+/// β play/resume. The icon flips βΆ β βΈ.+unsafe fn on_play_toggle(hwnd: HWND) {+ STATE.with(|s| {+ let mut b = s.borrow_mut(); let Some(st) = b.as_mut() else { return };+ if st.playing {+ // β pause: freeze position, show the βΆ (play/resume) icon.+ if let Some(t0) = st.play_start { st.accum_ms += t0.elapsed().as_millis() as u64; }+ st.playing = false; st.play_start = None;+ emit("PAUSE"); let _ = SetWindowTextW(st.hbtn, ICON_PLAY);+ } else {+ // β play/resume: show the βΈ (pause) icon.+ st.playing = true; st.play_start = Some(Instant::now());+ emit("PLAY"); let _ = SetWindowTextW(st.hbtn, ICON_PAUSE);+ }+ });+ let _ = InvalidateRect(hwnd, None, true);+}++/// Current song position in ms (accumulated finished segments + the running one).+fn position_ms(st: &State) -> u64 {+ st.accum_ms + st.play_start.map(|t| t.elapsed().as_millis() as u64).unwrap_or(0)+}++unsafe fn on_tick(hwnd: HWND) {+ // Returns the region to repaint this tick (None = nothing changed). Only the+ // changing band is invalidated, so an idle header/legend never re-composites+ // β that frees DWM/GPU bandwidth for the agent-cursor overlay (which is the+ // expensive full-virtual-screen layered window the cursors live on).+ let region: Option<RECT> = STATE.with(|s| {+ let mut b = s.borrow_mut(); let st = b.as_mut()?;+ let (cw, ch) = (st.cw.max(1), st.ch.max(1));+ match st.mode {+ Mode::Instrument => {+ let had = !st.pulses.is_empty() || st.pad_glow > 0.01+ || st.zone_glow.iter().any(|&z| z > 0.01);+ for p in st.pulses.iter_mut() { p.age += 0.040; p.inten *= 0.88; }+ st.pulses.retain(|p| p.inten > 0.03);+ st.pad_glow *= 0.88;+ if st.pad_glow < 0.01 { st.pad_glow = 0.0; }+ for z in st.zone_glow.iter_mut() { *z *= 0.88; if *z < 0.01 { *z = 0.0; } }+ had.then(|| fr(cw, ch, 0.0, WY0 - 0.03, 1.0, 1.0)) // widget band only+ }+ Mode::Controller => {+ if !st.playing { return None; }+ if position_ms(st) > st.dur_ms + 250 {+ // Song ended β reset to the top. Emit STOP so the orchestrator+ // clears its playing flag + resets its resume offset to 0.+ st.playing = false; st.play_start = None; st.accum_ms = 0;+ emit("STOP"); let _ = SetWindowTextW(st.hbtn, ICON_PLAY);+ return Some(fr(cw, ch, 0.0, 0.0, 1.0, 1.0)); // full repaint once+ }+ Some(fr(cw, ch, 0.0, 0.91, 1.0, 1.0)) // playhead bar only+ }+ }+ });+ if let Some(r) = region { let _ = InvalidateRect(hwnd, Some(&r), false); }+}++unsafe fn text(hdc: windows::Win32::Graphics::Gdi::HDC, r: RECT, s: &str, fmt: windows::Win32::Graphics::Gdi::DRAW_TEXT_FORMAT) {+ let mut t = wide(s); let mut rr = r;+ DrawTextW(hdc, &mut t, &mut rr, fmt | DT_SINGLELINE);+}++unsafe fn paint(hwnd: HWND) {+ let mut ps = PAINTSTRUCT::default();+ let hdc = BeginPaint(hwnd, &mut ps);+ STATE.with(|s| {+ let b = s.borrow(); let Some(st) = b.as_ref() else { return };+ let (cw, ch) = (st.cw.max(1), st.ch.max(1));+ // Double-buffer: build the whole frame in an off-screen DC, then blit it+ // to the window in one BitBlt. Painting straight to the window DC (with a+ // full-client background fill every frame, ~30fps from the glow/playhead+ // timer) is what caused the flicker β never the windows being recreated.+ let mem = CreateCompatibleDC(hdc);+ let bmp = CreateCompatibleBitmap(hdc, cw, ch);+ let old = SelectObject(mem, bmp);+ SetBkMode(mem, TRANSPARENT);++ let bg = CreateSolidBrush(rgb(0x0b, 0x0c, 0x12));+ let panel = CreateSolidBrush(rgb(0x16, 0x18, 0x22));+ let line = CreateSolidBrush(rgb(0x2a, 0x2d, 0x3e));+ let dim = rgb(0x6a, 0x6f, 0x85);+ let ink = rgb(0xe8, 0xea, 0xf2);+ let full = RECT { left: 0, top: 0, right: cw, bottom: ch };+ FillRect(mem, &full, bg);++ match st.mode {+ Mode::Controller => paint_controller(mem, st, cw, ch, &panel, &line, ink, dim),+ Mode::Instrument => paint_instrument(mem, st, cw, ch, &panel, &line, ink, dim),+ }+ for o in [bg, panel, line] { let _ = DeleteObject(o); }++ let _ = BitBlt(hdc, 0, 0, cw, ch, mem, 0, 0, SRCCOPY);+ SelectObject(mem, old);+ let _ = DeleteObject(bmp);+ let _ = DeleteDC(mem);+ });+ let _ = EndPaint(hwnd, &ps);+}++unsafe fn paint_controller(hdc: windows::Win32::Graphics::Gdi::HDC, st: &State, cw: i32, ch: i32, _panel: &HBRUSH, line: &HBRUSH, _ink: COLORREF, dim: COLORREF) {+ // Thin transport bar. A small icon play/pause button sits at the far left+ // (~9%); lay the rest out horizontally: title, bpm, track swatches, playhead.+ SelectObject(hdc, st.f_title); SetTextColor(hdc, rgb(0xff, 0xff, 0xff));+ text(hdc, fr(cw, ch, 0.11, 0.06, 0.55, 0.58), "CUA JUKEBOX", DT_LEFT | DT_VCENTER);+ SelectObject(hdc, st.f_label); SetTextColor(hdc, dim);+ text(hdc, fr(cw, ch, 0.115, 0.52, 0.55, 0.95),+ &format!("{} parts Β· {} bpm", st.tracks.len(), st.bpm), DT_LEFT | DT_VCENTER);++ // Row of track colour swatches (matches each instrument tile's colour).+ let n = st.tracks.len().max(1);+ let (x0, x1) = (0.56_f64, 0.985_f64);+ let sw_w = (x1 - x0) / n as f64;+ for (i, (_name, col)) in st.tracks.iter().enumerate() {+ let sx = x0 + i as f64 * sw_w;+ let sw = fr(cw, ch, sx, 0.18, sx + sw_w * 0.7, 0.66);+ let cb = CreateSolidBrush(*col); FillRect(hdc, &sw, cb); let _ = DeleteObject(cb);+ }++ // Playhead along the very bottom.+ let bar = fr(cw, ch, 0.0, 0.92, 1.0, 1.0);+ FillRect(hdc, &bar, *line);+ let frac = (position_ms(st) as f64 / st.dur_ms.max(1) as f64).clamp(0.0, 1.0);+ if frac > 0.0 {+ let mut fb = bar; fb.right = bar.left + ((bar.right - bar.left) as f64 * frac) as i32;+ // Dim while paused, bright while playing.+ let c = if st.playing { st.accent } else { mix((0x2a, 0x2d, 0x3e), st.accent_rgb, 0.45) };+ let cb = CreateSolidBrush(c); FillRect(hdc, &fb, cb); let _ = DeleteObject(cb);+ }+}++unsafe fn paint_instrument(hdc: windows::Win32::Graphics::Gdi::HDC, st: &State, cw: i32, ch: i32, _panel: &HBRUSH, line: &HBRUSH, ink: COLORREF, dim: COLORREF) {+ let black = (0x0b, 0x0c, 0x12);+ // header: swatch + label + hits+ let sw = fr(cw, ch, 0.06, 0.09, 0.10, 0.17);+ let cb = CreateSolidBrush(st.accent); FillRect(hdc, &sw, cb); let _ = DeleteObject(cb);+ SelectObject(hdc, st.f_title); SetTextColor(hdc, ink);+ text(hdc, fr(cw, ch, 0.13, 0.07, 0.78, 0.20), &st.label, DT_LEFT | DT_VCENTER);+ SelectObject(hdc, st.f_label); SetTextColor(hdc, dim);+ text(hdc, fr(cw, ch, 0.60, 0.07, 0.95, 0.20), &format!("{}βͺ", st.hits), DT_LEFT | DT_VCENTER);++ let frame = |r: &RECT, c: COLORREF| { let br = CreateSolidBrush(c); FrameRect(hdc, r, HBRUSH(br.0)); let _ = DeleteObject(br); };+ let panel_bg = (0x20, 0x23, 0x30);++ match st.kind {+ Kind::Pad => {+ // Brightness = the constantly-fading, impulse-driven pad_glow: fast+ // hits pile up to a bright pad, sparse hits let it dim between them.+ let g = st.pad_glow.clamp(0.0, 1.0);+ let pad = fr(cw, ch, WX0 + 0.10, WY0, WX1 - 0.10, WY1);+ let body = mix(panel_bg, st.accent_rgb, g * 0.9);+ let bb = CreateSolidBrush(body); FillRect(hdc, &pad, bb); let _ = DeleteObject(bb);+ frame(&pad, st.accent);+ SelectObject(hdc, st.f_big);+ SetTextColor(hdc, if g > 0.4 { rgb(black.0, black.1, black.2) } else { st.accent });+ let cap = match st.wave { Wave::Kick => "KICK", Wave::Snare => "SNARE", Wave::Hat => "HAT", _ => "PAD" };+ text(hdc, pad, cap, DT_CENTER | DT_VCENTER);+ }+ Kind::Keys => {+ let strip = fr(cw, ch, WX0, WY0, WX1, WY1);+ FillRect(hdc, &strip, *line);+ let span = st.span.max(1);+ let w = (strip.right - strip.left) as f64 / span as f64;+ for k in 0..span {+ let kx = strip.left + (k as f64 * w) as i32;+ let cell = RECT { left: kx + 1, top: strip.top + 1, right: kx + w as i32 - 1, bottom: strip.bottom - 1 };+ // Each cell glows by the brightest pulse on THAT key β so a chord+ // lights several cells at once, each fading independently.+ let inten = st.pulses.iter().filter(|p| p.key == k).map(|p| p.inten).fold(0.0_f32, f32::max);+ let c = if inten > 0.02 { mix(panel_bg, st.accent_rgb, inten) }+ else if k % 12 == 0 { rgb(0x20, 0x23, 0x30) } else { rgb(0x18, 0x1a, 0x24) };+ let bb = CreateSolidBrush(c); FillRect(hdc, &cell, bb); let _ = DeleteObject(bb);+ }+ frame(&strip, st.accent);+ SelectObject(hdc, st.f_label); SetTextColor(hdc, dim);+ text(hdc, fr(cw, ch, WX0, WY1 + 0.005, WX1, 0.99), "pitch β low Β· high βΊ", DT_CENTER | DT_VCENTER);+ }+ Kind::Drums => {+ // Three pads β KICK Β· SNARE Β· HAT β each its own impulse/fade glow.+ let caps = ["KICK", "SNARE", "HAT"];+ SelectObject(hdc, st.f_label);+ for z in 0..3usize {+ let zx0 = WX0 + z as f64 * (WX1 - WX0) / 3.0;+ let zx1 = WX0 + (z as f64 + 1.0) * (WX1 - WX0) / 3.0;+ let pad = fr(cw, ch, zx0 + 0.01, WY0, zx1 - 0.01, WY1);+ let g = st.zone_glow[z].clamp(0.0, 1.0);+ let body = mix(panel_bg, st.accent_rgb, g * 0.9);+ let bb = CreateSolidBrush(body); FillRect(hdc, &pad, bb); let _ = DeleteObject(bb);+ frame(&pad, st.accent);+ SetTextColor(hdc, if g > 0.4 { rgb(black.0, black.1, black.2) } else { st.accent });+ text(hdc, pad, caps[z], DT_CENTER | DT_VCENTER);+ }+ }+ }+}diff --git a/demo/jukebox/orchestrator/Cargo.toml b/demo/jukebox/orchestrator/Cargo.tomlnew file mode 100644index 0000000000..913282c5cf--- /dev/null+++ b/demo/jukebox/orchestrator/Cargo.toml@@ -0,0 +1,22 @@+[package]+name = "jukebox-orchestrator"+version = "0.1.0"+edition = "2021"++[[bin]]+name = "jukebox-orchestrator"+path = "src/main.rs"++[dependencies]+# Minimal, well-tested SMF parser so real multitrack .mid files Just Work.+midly = { version = "0.5", default-features = false, features = ["std"] }+windows = { version = "0.58", features = [+ "Win32_Foundation",+ "Win32_UI_WindowsAndMessaging",+ "Win32_Graphics_Gdi",+ "Win32_Graphics_Dwm",+ "Win32_System_JobObjects",+ "Win32_System_Threading",+ "Win32_Security",+ "Win32_Media",+] }diff --git a/demo/jukebox/orchestrator/src/main.rs b/demo/jukebox/orchestrator/src/main.rsnew file mode 100644index 0000000000..d1d3e1f55a--- /dev/null+++ b/demo/jukebox/orchestrator/src/main.rs@@ -0,0 +1,841 @@+//! CUA JUKEBOX orchestrator β coordinated multi-cursor "computer-use" music.+//!+//! Reads a MIDI file (or a built-in demo song), turns each track into its own+//! instrument window (a miniwob-style minigame), and gives each one its OWN+//! cua-driver session = its OWN uniquely-coloured agent cursor. While the song+//! plays, every note steers that track's cursor onto its widget and clicks it+//! in the background β the click is what makes the sound. One cursor per part,+//! one colour per agent, all driven off a single clock: the dumbest possible+//! orchestra, performed entirely by background computer-use.+//!+//! Timing is locked with the cursor's `glide_duration_ms` (set per session via+//! `set_agent_cursor_motion`): every glide takes a known, fixed time regardless+//! of how far the cursor must travel across its pitch strip, so the orchestrator+//! can pre-roll each click by exactly that lead and land the actuation on the+//! beat. (That field is honoured identically on macOS and Windows β it lives in+//! the shared render core.)++use std::path::PathBuf;+use std::process::{Child, Command, Stdio};+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::Arc;+use std::thread;+use std::time::{Duration, Instant};++use windows::Win32::Foundation::{BOOL, HANDLE, HWND, LPARAM, POINT, RECT, TRUE};+use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS};+use windows::Win32::Graphics::Gdi::ClientToScreen;+use windows::Win32::System::JobObjects::{+ AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject,+ JobObjectExtendedLimitInformation, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,+ JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,+};+use windows::Win32::UI::WindowsAndMessaging::{+ EnumWindows, GetClientRect, GetWindowTextW, GetWindowThreadProcessId, IsWindowVisible,+ SetForegroundWindow, SetWindowPos, HWND_TOP, SWP_NOACTIVATE, SWP_NOZORDER, SWP_SHOWWINDOW,+};++// Widget geometry β MUST match jukebox-app's WX0/WX1/WY0/WY1/KEYS_SPAN.+const WX0: f64 = 0.06;+const WX1: f64 = 0.94;+const WY0: f64 = 0.34;+const WY1: f64 = 0.92;+const KEYS_SPAN: i32 = 24;+const MAX_TRACKS: usize = 9;++// One cua-driver palette per agent. Keying a session to a palette NAME makes+// that session's overlay cursor render in that palette automatically+// (`Palette::for_instance(session)`), exactly like the sibling multi-cursor+// demo. The hex is that palette's mid colour, reused for the instrument window+// accent AND the transport legend, so the cursor, its window, and the legend+// all read as the same colour. (`cursor_color` on `set_agent_cursor_motion`+// only updates registry state, not the Windows overlay paint β so we rely on+// the session-name palette instead of fighting it.)+const SESSIONS: [(&str, &str); 9] = [+ ("crimson", "#e85262"),+ ("amber", "#f4b242"),+ ("aqua", "#4ccce0"),+ ("mint_lime", "#60daae"),+ ("orchid", "#dd71ec"),+ ("soft_purple", "#b284ff"),+ ("rose_gold", "#f784aa"),+ ("chartreuse", "#b8dc36"),+ ("cobalt", "#507eec"),+];++#[derive(Clone, Copy, PartialEq)]+enum Kind { Pad, Keys, Drums }++#[derive(Clone)]+struct Note {+ t: f64,+ pitch: u8,+ /// Carried from the source for fidelity; the instrument picks its own hit+ /// velocity (a background click can't carry one), so it's unused here.+ #[allow(dead_code)]+ vel: u8,+}++struct Track {+ name: String,+ notes: Vec<Note>,+ kind: Kind,+ wave: &'static str,+ root: i32,+ /// Number of semitones the pitch strip spans (fit to the track's range so+ /// wide melodies aren't clamped to the top key). Drums ignore it.+ span: i32,+ color: &'static str,+ session: String,+ title: String,+ hwnd: HWND,+ pid: u32,+ /// True when this track is percussion (most notes on MIDI channel 10, or a+ /// drum-kit name) β rendered as a 3-zone kick/snare/hat pad.+ is_drum: bool,+}++struct Song { tracks: Vec<Track>, bpm: u32, dur_sec: f64 }++/// Send-able snapshot of one cursor's worth of work (a "voice" β a single pool+/// member of a track). HWND is carried as an isize because the raw handle+/// pointer isn't `Send`. Several voices can share one track's window/colour.+struct Voice {+ pid: u32,+ hwnd_addr: isize,+ session: String,+ kind: Kind,+ root: i32,+ span: i32,+ notes: Arc<Vec<Note>>,+}++/// Max same-colour cursors a single track may grow to (a chord wider than this+/// reuses cursor 0 and accepts a little overlap rather than spawning forever).+const MAX_POOL: usize = 6;++/// Assign each (time-sorted) note to a pool cursor: try the lowest-indexed+/// cursor that's free (its previous glide finished β₯ now), else grow the pool,+/// else (at the cap) reuse cursor 0. `busy` is the glide window a cursor is+/// "occupied" for after it fires. Returns the per-note cursor index and the+/// resulting pool size. This is the deterministic form of "try the first, go to+/// the next if occupied during the 100ms, spawn one if none free".+fn assign_pool(notes: &[Note], busy: f64, cap: usize) -> (Vec<usize>, usize) {+ let mut free_at: Vec<f64> = Vec::new();+ let mut assign = Vec::with_capacity(notes.len());+ for n in notes {+ let chosen = free_at.iter().position(|&f| f <= n.t + 1e-6);+ let i = match chosen {+ Some(i) => i,+ None if free_at.len() < cap => { free_at.push(0.0); free_at.len() - 1 }+ None => 0,+ };+ free_at[i] = n.t + busy;+ assign.push(i);+ }+ (assign, free_at.len().max(1))+}++/// Blend a `#rrggbb` toward white by `t` (0=unchanged, 1=white) for a gradient+/// tip stop, so a forced-colour cursor still reads as a lit arrow, not a flat blob.+fn lighten(hex: &str, t: f64) -> String {+ let v = u32::from_str_radix(hex.trim_start_matches('#'), 16).unwrap_or(0xffffff);+ let f = |sh: u32| { let c = ((v >> sh) & 0xff) as f64; (c + (255.0 - c) * t) as u32 };+ format!("#{:02x}{:02x}{:02x}", f(16), f(8), f(0))+}++// ββ kill-on-exit job (whole tree dies with the orchestrator) βββββββββββββββββββ+static JOB: std::sync::OnceLock<usize> = std::sync::OnceLock::new();+fn job() -> HANDLE {+ let raw = *JOB.get_or_init(|| unsafe {+ let h = CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW");+ let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();+ info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;+ let _ = SetInformationJobObject(h, JobObjectExtendedLimitInformation,+ &info as *const _ as *const core::ffi::c_void,+ std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32);+ h.0 as usize+ });+ HANDLE(raw as *mut core::ffi::c_void)+}+fn assign_to_job(child: &Child) {+ use std::os::windows::io::AsRawHandle;+ unsafe { let _ = AssignProcessToJobObject(job(), HANDLE(child.as_raw_handle() as *mut core::ffi::c_void)); }+}++fn main() {+ let demo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf();+ let repo_root = demo_root.parent().unwrap().parent().unwrap().to_path_buf();++ let cua = std::env::var("CUA_DRIVER_EXE").map(PathBuf::from)+ .unwrap_or_else(|_| repo_root.join("libs/cua-driver/rust/target/debug/cua-driver.exe"));+ let app = std::env::var("JUKEBOX_APP_EXE").map(PathBuf::from)+ .unwrap_or_else(|_| demo_root.join("target/debug/jukebox-app.exe"));+ if !cua.exists() { eprintln!("cua-driver.exe not found at {cua:?}"); std::process::exit(1); }+ if !app.exists() { eprintln!("jukebox-app.exe not found at {app:?} β run `cargo build` first"); std::process::exit(1); }++ // Raise the system timer resolution to 1ms so the per-note `thread::sleep`+ // that schedules each click is accurate to ~1ms instead of Windows' default+ // ~15ms β the dominant residual timing jitter once the per-note process+ // spawn was removed. Process-global; the OS restores it on exit.+ unsafe { let _ = windows::Win32::Media::timeBeginPeriod(1); }++ let args: Vec<String> = std::env::args().collect();+ let auto = args.iter().any(|a| a == "--auto");+ let midi_path = args.iter().skip(1).find(|a| a.to_lowercase().ends_with(".mid") || a.to_lowercase().ends_with(".midi"));+ let glide_ms: u64 = std::env::var("JUKEBOX_GLIDE_MS").ok().and_then(|s| s.parse().ok()).unwrap_or(200);+ // The dominant actuation latency IS the glide, so default the pre-roll to it+ // (the per-voice EMA refines from there); overridable with JUKEBOX_LEAD_MS.+ let lead = Duration::from_millis(std::env::var("JUKEBOX_LEAD_MS").ok().and_then(|s| s.parse().ok()).unwrap_or(glide_ms));++ let mut song = match midi_path {+ Some(p) => match load_midi(p) { Ok(s) => s, Err(e) => { eprintln!("[orch] MIDI parse failed ({e}); using demo song"); demo_song() } },+ None => demo_song(),+ };+ assign_roles(&mut song);+ if song.tracks.is_empty() { eprintln!("[orch] no tracks with notes"); std::process::exit(1); }+ eprintln!("[orch] {} parts @ {} bpm, {:.1}s:", song.tracks.len(), song.bpm, song.dur_sec);+ for t in &song.tracks {+ let kind = match t.kind { Kind::Drums => "drums", Kind::Pad => "pad", Kind::Keys => "keys" };+ let lo = t.notes.iter().map(|n| n.pitch as i32).min().unwrap_or(0);+ let hi = t.notes.iter().map(|n| n.pitch as i32).max().unwrap_or(0);+ let range = hi - lo + 1;+ let fit = if t.kind == Kind::Keys && range > t.span {+ format!(" RANGE {range}st > strip {} β {} note(s) clamped", t.span,+ t.notes.iter().filter(|n| (n.pitch as i32 - t.root) >= t.span).count())+ } else { String::new() };+ eprintln!(" Β· {:<22} {:<6} {:<8} {:>4} notes range {}-{} ({}st), strip {}st{}",+ t.name, kind, t.wave, t.notes.len(), lo, hi, range, t.span, fit);+ }++ // Reap any leftover daemon from a previous run FIRST. Its job object only+ // fires KILL_ON_JOB_CLOSE when the *owning* orchestrator exits cleanly, so a+ // hard-killed run can leave a `cua-driver serve` alive β and a stale daemon+ // keeps its own full-screen overlay, whose agent0..N cursors stack on top of+ // this run's at the same (deterministic) coordinates. That's what makes it+ // look like more than one cursor per window. Start from a clean slate.+ for img in ["cua-driver.exe", "jukebox-app.exe"] {+ let _ = Command::new("taskkill").args(["/F", "/IM", img])+ .stdout(Stdio::null()).stderr(Stdio::null()).status();+ }+ // Give the killed daemon time to release its named pipe before we start a+ // fresh one β too short a wait races the pipe handoff and can wedge the new+ // daemon's first connections (observed as a one-off ~10s-late performance).+ thread::sleep(Duration::from_millis(800));++ // Start the cua-driver daemon.+ eprintln!("[orch] starting cua-driver daemonβ¦");+ let mut daemon = Command::new(&cua).arg("serve").stdout(Stdio::null()).stderr(Stdio::null())+ .spawn().expect("spawn cua-driver serve");+ assign_to_job(&daemon);+ thread::sleep(Duration::from_millis(1500));++ // Launch the controller (foreground) + one instrument window per track.+ let mut controller = Command::new(&app)+ .args(["controller", "--title", "CUA JUKEBOX β Transport",+ "--color", "#3bd0ff",+ "--bpm", &song.bpm.to_string(),+ "--dur-ms", &((song.dur_sec * 1000.0) as u64).to_string(),+ "--tracks", &song.tracks.iter().map(|t| format!("{}|{}", t.name, t.color)).collect::<Vec<_>>().join(",")])+ .stdout(Stdio::piped()).stderr(Stdio::null()).spawn().expect("spawn controller");+ assign_to_job(&controller);+ let controller_out = controller.stdout.take().unwrap();++ let mut kids: Vec<Child> = Vec::new();+ for t in &song.tracks {+ let mut c = Command::new(&app);+ c.args(["instrument",+ "--title", &t.title,+ "--kind", match t.kind { Kind::Keys => "keys", Kind::Drums => "drums", Kind::Pad => "pad" },+ "--wave", t.wave,+ "--color", t.color,+ "--label", &t.name,+ "--root", &t.root.to_string(),+ "--span", &t.span.to_string()]);+ if let Ok(ch) = c.stdout(Stdio::null()).stderr(Stdio::null()).spawn() { assign_to_job(&ch); kids.push(ch); }+ }++ thread::sleep(Duration::from_millis(1400)); // window warmup++ // Discover HWNDs/pids.+ for t in song.tracks.iter_mut() {+ if let Some((h, pid)) = find_window_by_title(&t.title) { t.hwnd = h; t.pid = pid; }+ else { eprintln!("[orch] (warn) no window for '{}'", t.title); }+ }+ song.tracks.retain(|t| !t.hwnd.0.is_null());+ let controller_hwnd = find_window_by_title("CUA JUKEBOX β Transport").map(|(h, _)| h);++ // Layout: a 600Γ80 transport bar above a tight grid of fixed 200Γ160 tiles,+ // packed GAP px apart and centered in the work area.+ const TILE_W: i32 = 200;+ const TILE_H: i32 = 160;+ const CTRL_W: i32 = 600;+ const CTRL_H: i32 = 80;+ const GAP: i32 = 6;+ let wa = work_area();+ let n = song.tracks.len() as i32;+ let cols = (n as f64).sqrt().ceil().max(1.0) as i32;+ let rows = (n + cols - 1) / cols.max(1);+ let grid_w = cols * TILE_W + (cols - 1) * GAP;+ let grid_h = rows * TILE_H + (rows - 1) * GAP;+ let block_w = grid_w.max(CTRL_W);+ let block_h = CTRL_H + GAP + grid_h;+ let ox = wa.left + (((wa.right - wa.left) - block_w) / 2).max(0);+ let oy = wa.top + (((wa.bottom - wa.top) - block_h) / 2).max(0);+ if let Some(ch) = controller_hwnd {+ place(ch, ox + (block_w - CTRL_W) / 2, oy, CTRL_W, CTRL_H, true);+ }+ let gy = oy + CTRL_H + GAP;+ let gx = ox + (block_w - grid_w) / 2;+ for (i, t) in song.tracks.iter().enumerate() {+ let (col, row) = (i as i32 % cols, i as i32 / cols);+ place(t.hwnd, gx + col * (TILE_W + GAP), gy + row * (TILE_H + GAP), TILE_W, TILE_H, false);+ }+ if let Some(ch) = controller_hwnd { unsafe { let _ = SetForegroundWindow(ch); } }+ thread::sleep(Duration::from_millis(700));++ // Build per-track cursor POOLS and arm each member. A track usually needs+ // one cursor, but when notes land within the glide window (e.g. a chord) the+ // first cursor is still mid-glide, so the next free pool cursor takes the+ // note β growing the pool. Every member of a track is forced to the SAME+ // colour (via set_agent_cursor_style's gradient β cursor_color on+ // set_agent_cursor_motion is registry-only on the Windows overlay and would+ // not repaint), so a chord fans out into several identically-coloured+ // cursors on that one window.+ // A cursor is "occupied" for the whole click β the glide PLUS the dispatch+ // overhead (IPC + click-post + arrival frame), not just the glide. Sizing+ // the pool against the real click duration means any track whose notes are+ // closer together than that spawns enough cursors to keep up, instead of one+ // cursor falling progressively behind.+ let busy_sec = (glide_ms as f64 + 130.0) / 1000.0;+ let mut voices: Vec<Voice> = Vec::new();+ for t in &song.tracks {+ let (assign, pool) = assign_pool(&t.notes, busy_sec, MAX_POOL);+ let tip = lighten(t.color, 0.5);+ for k in 0..pool {+ let session = format!("{}_{}", t.session, k);+ let _ = run_call(&cua, "set_agent_cursor_enabled",+ &format!(r#"{{"enabled":true,"session":"{session}"}}"#));+ // Fixed 100ms glide so every actuation lands on the beat regardless+ // of travel distance (and so "occupied for the glide window" has a+ // known length the pool sizes against).+ // turn_radius 40 = half the default 80 β tighter glide curves, which+ // read better in the small tiles.+ let _ = run_call(&cua, "set_agent_cursor_motion", &format!(+ r#"{{"session":"{session}","cursor_label":"{}","cursor_size":15,"glide_duration_ms":{glide_ms},"spring":1.0,"arc_size":0.08,"dwell_after_click_ms":0,"idle_hide_ms":0,"turn_radius":40}}"#,+ t.name));+ let _ = run_call(&cua, "set_agent_cursor_style", &format!(+ r#"{{"session":"{session}","gradient_colors":["{tip}","{}"],"bloom_color":"{}"}}"#,+ t.color, t.color));+ let notes_k: Vec<Note> = t.notes.iter().zip(assign.iter())+ .filter(|(_, &a)| a == k).map(|(n, _)| n.clone()).collect();+ voices.push(Voice {+ pid: t.pid,+ hwnd_addr: t.hwnd.0 as isize,+ session,+ kind: t.kind,+ root: t.root,+ span: t.span,+ notes: Arc::new(notes_k),+ });+ }+ if pool > 1 {+ eprintln!("[orch] {:<10} pool={} same-colour cursors (notes closer than {:.0}ms)", t.name, pool, busy_sec * 1000.0);+ }+ }+ let plans: Arc<Vec<Voice>> = Arc::new(voices);++ let cua = Arc::new(cua);+ let playing = Arc::new(AtomicBool::new(false));+ let generation = Arc::new(AtomicU64::new(0));+ let dur_sec = song.dur_sec;+ let base_lead_ms = lead.as_secs_f64() * 1000.0;+ // `timing` collects the signed per-note error (actualβscheduled) for the+ // end-of-song report; cleared each performance. The adaptive lead itself is+ // PER-VOICE (a thread-local EMA below), so one congested track β e.g. the+ // Pad pool firing a triad β can't skew the lead of the tight single-cursor+ // tracks.+ let timing = Arc::new(std::sync::Mutex::new(Vec::<f64>::new()));+ // Pause/resume: `offset` is the song-second to (re)start from; `seg` records+ // the wall-clock start + offset of the running segment so PAUSE can compute+ // where we are. STOP resets offset to 0; PAUSE saves the current position.+ let offset = Arc::new(std::sync::Mutex::new(0.0_f64));+ let seg = Arc::new(std::sync::Mutex::new((Instant::now(), 0.0_f64)));++ let start_perf = {+ let plans = plans.clone();+ let cua = cua.clone();+ let playing = playing.clone();+ let generation = generation.clone();+ let timing = timing.clone();+ let seg = seg.clone();+ move |off: f64| {+ playing.store(true, Ordering::SeqCst);+ let g = generation.fetch_add(1, Ordering::SeqCst) + 1;+ let start = Instant::now();+ *seg.lock().unwrap() = (start, off);+ timing.lock().unwrap().clear();+ eprintln!("[orch] βΆ performing from {off:.1}s β {} cursors actuating in the background", plans.len());+ for ti in 0..plans.len() {+ let plans = plans.clone();+ let cua = cua.clone();+ let playing = playing.clone();+ let generation = generation.clone();+ let timing = timing.clone();+ thread::spawn(move || {+ let p = &plans[ti];+ let hwnd = HWND(p.hwnd_addr as *mut core::ffi::c_void);+ // Thread-local adaptive lead (ms) for THIS voice only,+ // warm-started at the ~constant dispatch overhead on top of+ // the glide (IPC + click-post + the arrival frame) so even+ // the first notes land near the beat; the EMA refines it.+ let mut corr = 110.0_f64;+ // One persistent daemon connection for this whole track β+ // pipelines every click with no per-note process spawn (that+ // spawn was the entire timing-jitter source). The daemon runs+ // a handler task per connection, so the 6 tracks actuate+ // concurrently; within a track, clicks are lock-step (the+ // response marks the actuation moment we measure against).+ let pipe = std::env::var("CUA_DRIVER_PIPE")+ .unwrap_or_else(|_| r"\\.\pipe\cua-driver".into());+ let mut conn = DaemonConn::open(&pipe);+ for (idx, note) in p.notes.iter().enumerate() {+ if !playing.load(Ordering::SeqCst) || generation.load(Ordering::SeqCst) != g { return; }+ if note.t < off { continue; } // already played before the resume point+ // Fire early by the adaptive lead so the actuation lands+ // on the note's scheduled time despite the glide latency.+ // `start` represents song-time `off`, so schedule at the+ // note's time minus that offset.+ let eff_lead = (base_lead_ms + corr).max(0.0);+ let due = start + Duration::from_secs_f64(note.t - off);+ let fire = due.checked_sub(Duration::from_secs_f64(eff_lead / 1000.0)).unwrap_or(due);+ let now = Instant::now();+ if fire > now { thread::sleep(fire - now); }+ if !playing.load(Ordering::SeqCst) || generation.load(Ordering::SeqCst) != g { return; }+ let (xf, yf) = target(p, idx, note.pitch);+ if let Some((x, y)) = client_rel_to_local_px(hwnd, xf, yf) {+ let args = format!(+ r#"{{"pid":{},"window_id":{},"x":{},"y":{},"session":"{}"}}"#,+ p.pid, p.hwnd_addr, x, y, p.session);+ let ok = match conn.as_mut() { Some(c) => c.call("click", &args), None => false };+ if !ok {+ // Reconnect once, else fall back to a one-shot call.+ conn = DaemonConn::open(&pipe);+ match conn.as_mut() {+ Some(c) => { let _ = c.call("click", &args); }+ None => { let _ = run_call(&cua, "click", &args); }+ }+ }+ // Response received β the click landed (actuation moment).+ let err_ms = (start.elapsed().as_secs_f64() - note.t) * 1000.0;+ timing.lock().unwrap().push(err_ms);+ // EMA drives the mean error to zero: eff_lead = base ++ // corr and err = latency β eff_lead, so corr += Ξ±Β·err+ // converges corr β latency β base. Clamp tight so a+ // transient stall can't integrate the lead away. Ξ± is+ // fairly high so short voices (a few-note pool member)+ // converge within the song.+ corr = (corr + 0.30 * err_ms).clamp(-base_lead_ms, 600.0);+ }+ }+ });+ }+ // End-of-song timing report (one per performance/generation).+ let timing = timing.clone();+ let generation = generation.clone();+ thread::spawn(move || {+ thread::sleep(Duration::from_secs_f64((dur_sec - off).max(0.0) + 0.8));+ if generation.load(Ordering::SeqCst) != g { return; }+ report_timing(&timing.lock().unwrap(), base_lead_ms);+ });+ }+ };++ if auto {+ thread::sleep(Duration::from_millis(800));+ start_perf(0.0);+ } else {+ eprintln!("[orch] ready β click βΆ PLAY in the Transport window to start the performance.");+ }++ // React to the transport's PLAY / PAUSE / STOP (the human's one action that+ // drives the whole coordinated fleet β like the master window in the sibling+ // demo). PLAY resumes from the held offset; PAUSE freezes the position; STOP+ // resets to the top.+ use std::io::{BufRead, BufReader};+ let reader = BufReader::new(controller_out);+ for line in reader.lines().map_while(Result::ok) {+ // Drag-drop a .mid onto the transport β restart the whole demo with that+ // track. Re-exec a fresh orchestrator (detached, outside our job object,+ // so it survives our exit); it reaps our daemon/windows on startup and+ // comes up READY on the dropped file (no --auto β the user presses βΆ).+ if let Some(path) = line.trim_end().strip_prefix("LOAD\t") {+ eprintln!("[orch] β» restarting with {path}");+ if let Ok(exe) = std::env::current_exe() {+ let _ = Command::new(exe).arg(path)+ .stdout(Stdio::null()).stderr(Stdio::null()).spawn();+ }+ break; // fall through to teardown; the new orchestrator takes over+ }+ match line.trim() {+ // Idempotent: ignore PLAY while already performing (resume still works+ // β PAUSE clears `playing` first), so a stray PLAY can't double-start.+ "PLAY" if !playing.load(Ordering::SeqCst) => { let off = *offset.lock().unwrap(); start_perf(off); }+ "PAUSE" => {+ playing.store(false, Ordering::SeqCst);+ let (rs, off) = *seg.lock().unwrap();+ let pos = off + rs.elapsed().as_secs_f64();+ *offset.lock().unwrap() = pos;+ eprintln!("[orch] βΈ paused at {pos:.1}s");+ }+ "STOP" => {+ playing.store(false, Ordering::SeqCst);+ *offset.lock().unwrap() = 0.0;+ eprintln!("[orch] β stopped");+ }+ _ => {}+ }+ }++ // Controller closed β tear everything down (job object kills the tree too).+ playing.store(false, Ordering::SeqCst);+ let _ = controller.kill();+ for mut k in kids { let _ = k.kill(); }+ let _ = daemon.kill();+}++// ββ song construction ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++/// True if a track name reads like a full drum kit (vs a single drum part like+/// "Kick", which stays a single pad).+fn name_is_kit(name: &str) -> bool {+ let n = name.to_lowercase();+ ["drum kit", "drumkit", "drums", "percussion", "drum set"].iter().any(|s| n.contains(s))+}++/// Map a MIDI channel-10 drum note to a 3-zone pad: 0=kick, 1=snare/clap/tom,+/// 2=hat/cymbal/perc. (General MIDI percussion key map.)+fn drum_zone(pitch: u8) -> i32 {+ match pitch {+ 35 | 36 => 0, // bass/kick drums+ 37 | 38 | 39 | 40 | 41 | 43 | 45 | 47 | 48 | 50 => 1, // snare, clap, rim, toms+ _ => 2, // hats, cymbals, perc+ }+}++/// Infer a melodic track's minigame + synth voice from its (lower-cased) name.+/// Wave keywords (square/saw/triangle/8-bit/chip/synth/scifi/β¦) are matched+/// explicitly so chiptune/GM names map to a fitting voice instead of plain sine.+fn infer(name: &str) -> (Kind, &'static str) {+ let n = name.to_lowercase();+ let has = |k: &[&str]| k.iter().any(|s| n.contains(s));+ // Single drum parts (not full kits) β one-shot pad.+ if has(&["kick", "bass drum", "bd"]) { (Kind::Pad, "kick") }+ else if has(&["snare", "clap", "sd"]) { (Kind::Pad, "snare") }+ else if has(&["hi-hat", "hihat", "hat", "cymbal", "ride", "shaker", "tom "]) { (Kind::Pad, "hat") }+ // Bass before the wave words so "Bass Guitar" is a bass, not a saw lead.+ else if has(&["bass"]) { (Kind::Keys, "saw") }+ // Explicit waveform names FIRST (so "8-Bit Triangle" is a triangle, not+ // caught by the chipβsquare fallback below).+ else if has(&["sawtooth", "saw"]) { (Kind::Keys, "saw") }+ else if has(&["triangle"]) { (Kind::Keys, "triangle") }+ else if has(&["square", "pulse"]) { (Kind::Keys, "square") }+ // Generic chiptune β square.+ else if has(&["8-bit", "8bit", "chip", "nes"]) { (Kind::Keys, "square") }+ // Timbre families.+ else if has(&["lead", "synth", "scifi", "sci-fi", "guitar", "trumpet", "sax", "brass", "pluck"]) { (Kind::Keys, "square") }+ else if has(&["pad", "string", "choir", "organ", "ambient", "smooth", "warm"]) { (Kind::Keys, "triangle") }+ else if has(&["arp", "bell", "key", "piano", "mallet", "celesta", "harp"]) { (Kind::Keys, "triangle") }+ else { (Kind::Keys, "sine") }+}++fn assign_roles(song: &mut Song) {+ song.tracks.truncate(MAX_TRACKS);+ for (i, t) in song.tracks.iter_mut().enumerate() {+ let (kind, wave) = if t.is_drum { (Kind::Drums, "kick") } else { infer(&t.name) };+ t.kind = kind;+ t.wave = wave;+ let (sess, hex) = SESSIONS[i % SESSIONS.len()];+ t.color = hex;+ t.session = sess.to_string();+ t.title = format!("JUKEBOX {i:02} β {}", t.name);+ // Fit the pitch strip to the track's actual range so wide melodies are+ // played accurately instead of being clamped to the top key: root = the+ // lowest note, span = the full range (min 12 semitones for a usable+ // strip, capped at 48 = 4 octaves so the keys don't get microscopic).+ let lo = t.notes.iter().map(|n| n.pitch as i32).min().unwrap_or(48);+ let hi = t.notes.iter().map(|n| n.pitch as i32).max().unwrap_or(72);+ t.root = lo;+ t.span = (hi - lo + 1).clamp(12, 48);+ }+}++fn mk_track(name: &str, notes: Vec<Note>) -> Track {+ Track { name: name.into(), notes, kind: Kind::Keys, wave: "sine", root: 48, span: KEYS_SPAN,+ color: "#888", session: String::new(), title: String::new(), hwnd: HWND::default(), pid: 0,+ is_drum: false }+}++/// Generated 8-bar, 6-part loop at 120 bpm β a 1:1 feel-port of the HTML demo+/// song so the grid moves immediately even with no .mid file.+fn demo_song() -> Song {+ let q = 0.5_f64; // quarter-note seconds at 120 bpm+ let bars = 8;+ let roots = [36, 36, 43, 41];+ let sc = [0, 2, 4, 7, 9, 12];+ let (mut bass, mut kick, mut hat, mut pad, mut arp, mut lead) =+ (vec![], vec![], vec![], vec![], vec![], vec![]);+ for b in 0..bars {+ let o = b as f64 * q * 4.0;+ let r = roots[b % 4] as u8;+ bass.push(Note { t: o, pitch: r, vel: 100 });+ bass.push(Note { t: o + q, pitch: r, vel: 80 });+ bass.push(Note { t: o + 2.0 * q, pitch: r + 7, vel: 95 });+ bass.push(Note { t: o + 3.0 * q, pitch: r, vel: 80 });+ for k in 0..4 { kick.push(Note { t: o + k as f64 * q, pitch: 36, vel: 110 }); }+ for h in 0..8 { hat.push(Note { t: o + h as f64 * q / 2.0, pitch: 42, vel: if h % 2 == 0 { 80 } else { 55 } }); }+ // Triads struck simultaneously β three notes at the same instant means+ // one cursor can't cover them within the 100ms glide, so the Pad track+ // grows a 3-cursor same-colour pool that fans out across the strip.+ for iv in [12, 16, 19] { pad.push(Note { t: o, pitch: r + iv, vel: 52 }); }+ for iv in [12, 15, 19] { pad.push(Note { t: o + 2.0 * q, pitch: r + iv, vel: 50 }); }+ for a in 0..8 { arp.push(Note { t: o + a as f64 * q / 2.0, pitch: r + 24 + sc[(a + b) % 6] as u8, vel: 70 }); }+ if b % 2 == 1 { for l in 0..4 { lead.push(Note { t: o + l as f64 * q, pitch: r + 24 + sc[(l * 2) % 6] as u8, vel: 88 }); } }+ }+ Song {+ tracks: vec![+ mk_track("Bass", bass), mk_track("Kick", kick), mk_track("Hat", hat),+ mk_track("Pad", pad), mk_track("Arp", arp), mk_track("Lead", lead),+ ],+ bpm: 120,+ dur_sec: bars as f64 * q * 4.0,+ }+}++/// Parse a Standard MIDI File into per-track note lists. Single tempo (first+/// tempo event wins) β good enough for the demo; documented in the README.+fn load_midi(path: &str) -> Result<Song, String> {+ use midly::{MetaMessage, MidiMessage, Smf, Timing, TrackEventKind};+ let data = std::fs::read(path).map_err(|e| e.to_string())?;+ let smf = Smf::parse(&data).map_err(|e| e.to_string())?;+ let ppq = match smf.header.timing { Timing::Metrical(t) => t.as_int() as f64, _ => 480.0 };+ // First tempo found across all tracks (us per quarter-note).+ let mut tempo = 500_000.0_f64;+ 'outer: for tr in &smf.tracks {+ for ev in tr {+ if let TrackEventKind::Meta(MetaMessage::Tempo(t)) = ev.kind { tempo = t.as_int() as f64; break 'outer; }+ }+ }+ let spt = (tempo / 1e6) / ppq; // seconds per tick+ let mut tracks = Vec::new();+ let mut dur = 0.0_f64;+ for tr in &smf.tracks {+ let mut tick = 0u64;+ let mut name = String::new();+ // Key by (channel, pitch) so overlapping same-pitch notes on different+ // channels don't collide, and polyphonic same-pitch (rare) is handled.+ let mut on: std::collections::HashMap<(u8, u8), (f64, u8)> = std::collections::HashMap::new();+ let mut notes = Vec::new();+ let mut drum_notes = 0usize; // notes seen on MIDI channel 10 (index 9)+ for ev in tr {+ tick += ev.delta.as_int() as u64;+ let now = tick as f64 * spt;+ match ev.kind {+ TrackEventKind::Meta(MetaMessage::TrackName(bytes)) =>+ if name.is_empty() { name = String::from_utf8_lossy(bytes).trim().to_string(); },+ TrackEventKind::Midi { channel, message: MidiMessage::NoteOn { key, vel } } => {+ let ch = channel.as_int();+ let pitch = key.as_int();+ if vel.as_int() > 0 {+ on.insert((ch, pitch), (now, vel.as_int()));+ if ch == 9 { drum_notes += 1; }+ }+ else if let Some((t0, v)) = on.remove(&(ch, pitch)) { notes.push(Note { t: t0, pitch, vel: v }); dur = dur.max(now); }+ }+ TrackEventKind::Midi { channel, message: MidiMessage::NoteOff { key, .. } } => {+ if let Some((t0, v)) = on.remove(&(channel.as_int(), key.as_int())) { notes.push(Note { t: t0, pitch: key.as_int(), vel: v }); dur = dur.max(now); }+ }+ _ => {}+ }+ }+ if !notes.is_empty() {+ notes.sort_by(|a, b| a.t.partial_cmp(&b.t).unwrap());+ let idx = tracks.len();+ let nm = if name.is_empty() { format!("Track {}", idx + 1) } else { name.clone() };+ // Percussion if most notes were on channel 10, or the name is a kit.+ let is_drum = drum_notes * 2 > notes.len() || name_is_kit(&nm);+ let mut t = mk_track(&nm, notes);+ t.is_drum = is_drum;+ tracks.push(t);+ }+ }+ if tracks.is_empty() { return Err("no note tracks".into()); }+ let bpm = (60.0 / (tempo / 1e6)).round() as u32;+ Ok(Song { tracks, bpm, dur_sec: dur + 0.5 })+}++// ββ cua-driver call + Win32 helpers βββββββββββββββββββββββββββββββββββββββββββββ++/// Print the per-note timing diff: how far each actuation landed from its+/// scheduled beat (actual β scheduled, ms). `mean`β0 means the adaptive lead+/// has cancelled the systematic latency; `sd`/`p90` are the residual jitter+/// (dominated by per-note `cua-driver call` subprocess spawn).+fn report_timing(errs: &[f64], base_lead_ms: f64) {+ if errs.is_empty() { eprintln!("[timing] no notes measured"); return; }+ let n = errs.len();+ let mut s = errs.to_vec();+ s.sort_by(|a, b| a.partial_cmp(b).unwrap());+ let mean = errs.iter().sum::<f64>() / n as f64;+ let sd = (errs.iter().map(|e| (e - mean).powi(2)).sum::<f64>() / n as f64).sqrt();+ let absmean = errs.iter().map(|e| e.abs()).sum::<f64>() / n as f64;+ let pct = |q: f64| s[((n as f64 * q) as usize).min(n - 1)];+ eprintln!(+ "[timing] n={n} mean={mean:+.1}ms |mean|={absmean:.1}ms sd={sd:.1}ms \+ median={:+.1}ms p10={:+.1} p90={:+.1} min={:+.1} max={:+.1}",+ pct(0.5), pct(0.1), pct(0.9), s[0], s[n - 1],+ );+ eprintln!(+ "[timing] base lead {:.0}ms, refined per-voice; + β late, β β early.",+ base_lead_ms,+ );+}++fn run_call(cua: &PathBuf, tool: &str, json: &str) -> bool {+ Command::new(cua).arg("call").arg(tool).arg(json)+ .stdout(Stdio::null()).stderr(Stdio::null())+ .status().map(|s| s.success()).unwrap_or(false)+}++/// A persistent connection to the `cua-driver serve` daemon over its+/// line-delimited-JSON named pipe (`\\.\pipe\cua-driver`). The daemon handles+/// many requests per connection, so holding ONE open per track lets us pipeline+/// every note's click without paying a `cua-driver call` process spawn per note+/// β that spawn (tens of ms, high variance) was the entire timing-jitter+/// source. One connection per track also means tracks dispatch concurrently+/// server-side (a handler task per connection).+struct DaemonConn {+ writer: std::fs::File,+ reader: std::io::BufReader<std::fs::File>,+}++impl DaemonConn {+ fn open(pipe: &str) -> Option<DaemonConn> {+ use std::time::Instant as I;+ let deadline = I::now() + Duration::from_secs(3);+ loop {+ match std::fs::OpenOptions::new().read(true).write(true).open(pipe) {+ Ok(f) => {+ let reader = std::io::BufReader::new(f.try_clone().ok()?);+ return Some(DaemonConn { writer: f, reader });+ }+ Err(_) if I::now() < deadline => thread::sleep(Duration::from_millis(40)),+ Err(_) => return None,+ }+ }+ }++ /// Send one `call` and block for its response line. `args` is a JSON object+ /// literal. Returns whether the tool reported ok. The daemon processes one+ /// request per connection at a time and replies in order, so this lockstep+ /// is safe and the response marks the actuation moment.+ fn call(&mut self, tool: &str, args: &str) -> bool {+ use std::io::{BufRead, Write};+ let line = format!(r#"{{"method":"call","name":"{tool}","args":{args}}}"#) + "\n";+ if self.writer.write_all(line.as_bytes()).is_err() { return false; }+ if self.writer.flush().is_err() { return false; }+ let mut resp = String::new();+ match self.reader.read_line(&mut resp) {+ Ok(0) | Err(_) => false,+ Ok(_) => resp.contains("\"ok\":true"),+ }+ }+}++/// Window-local (x,y) fraction the cursor should click for a note. Melodic+/// strips map pitch β key cell (fixed y). Drum pads have no pitch, so we hop the+/// click around the pad each hit (deterministic per note index) β otherwise the+/// cursor would click dead-centre every time and look static; the hop makes the+/// 100ms glide visible. Must match jukebox-app's WX0/WX1/WY0/WY1/KEYS_SPAN.+fn target(p: &Voice, idx: usize, pitch: u8) -> (f64, f64) {+ match p.kind {+ Kind::Keys => {+ let span = p.span.max(1);+ let semi = (pitch as i32 - p.root).clamp(0, span - 1);+ (WX0 + (semi as f64 + 0.5) / span as f64 * (WX1 - WX0), (WY0 + WY1) / 2.0)+ }+ Kind::Drums => {+ // Click the kick / snare / hat zone this drum note belongs to.+ let z = drum_zone(pitch);+ (WX0 + (z as f64 + 0.5) / 3.0 * (WX1 - WX0), (WY0 + WY1) / 2.0)+ }+ Kind::Pad => {+ let h = (idx as u32).wrapping_mul(2_654_435_761);+ let xf = 0.30 + (h % 1000) as f64 / 1000.0 * 0.40;+ let yf = WY0 + 0.10 + ((h / 1000) % 1000) as f64 / 1000.0 * (WY1 - WY0 - 0.20);+ (xf, yf)+ }+ }+}++fn work_area() -> RECT {+ use windows::Win32::UI::WindowsAndMessaging::{SystemParametersInfoW, SPI_GETWORKAREA, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS};+ let mut wa = RECT::default();+ unsafe { let _ = SystemParametersInfoW(SPI_GETWORKAREA, 0, Some(&mut wa as *mut _ as *mut core::ffi::c_void), SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0)); }+ if wa.right <= wa.left || wa.bottom <= wa.top {+ use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};+ unsafe { wa = RECT { left: 0, top: 0, right: GetSystemMetrics(SM_CXSCREEN), bottom: GetSystemMetrics(SM_CYSCREEN) }; }+ }+ wa+}++fn place(hwnd: HWND, x: i32, y: i32, w: i32, h: i32, activate: bool) {+ if hwnd.0.is_null() { return; }+ let flags = if activate { SWP_SHOWWINDOW } else { SWP_NOACTIVATE | SWP_SHOWWINDOW | SWP_NOZORDER };+ unsafe { let _ = SetWindowPos(hwnd, HWND_TOP, x, y, w, h, flags); }+}++/// Client-relative (0..1) β the click tool's window-local screenshot-pixel space+/// (ClientToScreen minus the DWM extended-frame top-left + 1px inset). Same math+/// as the multi-cursor demo.+fn client_rel_to_local_px(hwnd: HWND, rx: f64, ry: f64) -> Option<(i32, i32)> {+ unsafe {+ let mut cr = RECT::default();+ GetClientRect(hwnd, &mut cr).ok()?;+ let mut pt = POINT { x: (rx * (cr.right - cr.left) as f64) as i32, y: (ry * (cr.bottom - cr.top) as f64) as i32 };+ let _ = ClientToScreen(hwnd, &mut pt);+ let mut dwm = RECT::default();+ if DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &mut dwm as *mut _ as *mut core::ffi::c_void, std::mem::size_of::<RECT>() as u32).is_err() {+ return Some((pt.x, pt.y));+ }+ Some((pt.x - dwm.left - 1, pt.y - dwm.top - 1))+ }+}++// ββ window discovery by title substring (case-insensitive) ββββββββββββββββββββββ+struct Finder { needle: String, hwnd: HWND, pid: u32 }+unsafe extern "system" fn enum_cb(hwnd: HWND, lparam: LPARAM) -> BOOL {+ let f = &mut *(lparam.0 as *mut Finder);+ if !IsWindowVisible(hwnd).as_bool() { return TRUE; }+ let mut buf = [0u16; 256];+ let n = GetWindowTextW(hwnd, &mut buf);+ if n > 0 {+ let title = String::from_utf16_lossy(&buf[..n as usize]);+ if title.to_lowercase().contains(&f.needle.to_lowercase()) {+ let mut pid = 0u32;+ GetWindowThreadProcessId(hwnd, Some(&mut pid));+ f.hwnd = hwnd; f.pid = pid;+ return BOOL(0);+ }+ }+ TRUE+}+fn find_window_by_title(needle: &str) -> Option<(HWND, u32)> {+ let deadline = Instant::now() + Duration::from_secs(8);+ loop {+ let mut f = Finder { needle: needle.to_string(), hwnd: HWND::default(), pid: 0 };+ unsafe { let _ = EnumWindows(Some(enum_cb), LPARAM(&mut f as *mut _ as isize)); }+ if !f.hwnd.0.is_null() { return Some((f.hwnd, f.pid)); }+ if Instant::now() > deadline { return None; }+ thread::sleep(Duration::from_millis(300));+ }+}diff --git a/demo/multi-cursor/.gitignore b/demo/multi-cursor/.gitignorenew file mode 100644index 0000000000..bbbbe20a0e--- /dev/null+++ b/demo/multi-cursor/.gitignore@@ -0,0 +1,8 @@+# Rust+/target/+# .NET+**/bin/+**/obj/+# Electron / Node+electron/node_modules/+electron/package-lock.jsondiff --git a/demo/multi-cursor/Cargo.lock b/demo/multi-cursor/Cargo.locknew file mode 100644index 0000000000..8df441af11--- /dev/null+++ b/demo/multi-cursor/Cargo.lock@@ -0,0 +1,312 @@+# This file is automatically @generated by Cargo.+# It is not intended for manual editing.+version = 4++[[package]]+name = "adler2"+version = "2.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"++[[package]]+name = "autocfg"+version = "1.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"++[[package]]+name = "bitflags"+version = "2.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"++[[package]]+name = "bytemuck"+version = "1.25.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"++[[package]]+name = "byteorder-lite"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"++[[package]]+name = "cfg-if"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"++[[package]]+name = "crc32fast"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"+dependencies = [+ "cfg-if",+]++[[package]]+name = "fdeflate"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"+dependencies = [+ "simd-adler32",+]++[[package]]+name = "flate2"+version = "1.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"+dependencies = [+ "crc32fast",+ "miniz_oxide",+]++[[package]]+name = "image"+version = "0.25.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"+dependencies = [+ "bytemuck",+ "byteorder-lite",+ "moxcms",+ "num-traits",+ "png",+]++[[package]]+name = "legacy-app"+version = "0.1.0"+dependencies = [+ "windows",+]++[[package]]+name = "miniz_oxide"+version = "0.8.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"+dependencies = [+ "adler2",+ "simd-adler32",+]++[[package]]+name = "moxcms"+version = "0.8.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"+dependencies = [+ "num-traits",+ "pxfm",+]++[[package]]+name = "num-traits"+version = "0.2.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"+dependencies = [+ "autocfg",+]++[[package]]+name = "orchestrator"+version = "0.1.0"+dependencies = [+ "image",+ "windows",+]++[[package]]+name = "png"+version = "0.18.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61"+dependencies = [+ "bitflags",+ "crc32fast",+ "fdeflate",+ "flate2",+ "miniz_oxide",+]++[[package]]+name = "proc-macro2"+version = "1.0.106"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"+dependencies = [+ "unicode-ident",+]++[[package]]+name = "pxfm"+version = "0.1.29"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f"++[[package]]+name = "quote"+version = "1.0.45"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"+dependencies = [+ "proc-macro2",+]++[[package]]+name = "simd-adler32"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"++[[package]]+name = "syn"+version = "2.0.117"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "unicode-ident"+version = "1.0.24"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"++[[package]]+name = "windows"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6"+dependencies = [+ "windows-core",+ "windows-targets",+]++[[package]]+name = "windows-core"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99"+dependencies = [+ "windows-implement",+ "windows-interface",+ "windows-result",+ "windows-strings",+ "windows-targets",+]++[[package]]+name = "windows-implement"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-interface"+version = "0.58.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515"+dependencies = [+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "windows-result"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"+dependencies = [+ "windows-targets",+]++[[package]]+name = "windows-strings"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"+dependencies = [+ "windows-result",+ "windows-targets",+]++[[package]]+name = "windows-targets"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"+dependencies = [+ "windows_aarch64_gnullvm",+ "windows_aarch64_msvc",+ "windows_i686_gnu",+ "windows_i686_gnullvm",+ "windows_i686_msvc",+ "windows_x86_64_gnu",+ "windows_x86_64_gnullvm",+ "windows_x86_64_msvc",+]++[[package]]+name = "windows_aarch64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"++[[package]]+name = "windows_aarch64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"++[[package]]+name = "windows_i686_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"++[[package]]+name = "windows_i686_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"++[[package]]+name = "windows_i686_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"++[[package]]+name = "windows_x86_64_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"++[[package]]+name = "windows_x86_64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"++[[package]]+name = "windows_x86_64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"diff --git a/demo/multi-cursor/Cargo.toml b/demo/multi-cursor/Cargo.tomlnew file mode 100644index 0000000000..bf84427611--- /dev/null+++ b/demo/multi-cursor/Cargo.toml@@ -0,0 +1,6 @@+[workspace]+resolver = "2"+members = ["legacy-app", "orchestrator"]++[profile.release]+opt-level = 2diff --git a/demo/multi-cursor/README.md b/demo/multi-cursor/README.mdnew file mode 100644index 0000000000..496fee1387--- /dev/null+++ b/demo/multi-cursor/README.md@@ -0,0 +1,89 @@+# Multi-cursor background computer-use demo β "National Records System"++A fleet of deliberately **legacy-looking government records terminals** (navy+banner, `UNCLASSIFIED // FOR OFFICIAL USE ONLY` strip, function-key bar,+green-screen records grid, status line) β the kind of internal agency app that,+in the age of AI, has *no* automation integration. cua-driver automates them+anyway.++One human action in the foreground "master" terminal is replayed onto **four+background terminals at the same time**, each driven by its own cua-driver+session = its own uniquely-coloured agent cursor β with **no window ever+raised** and **the user's mouse never moved**.++It also proves cua-driver works **with or without an accessibility tree**: the+five windows span five UI frameworks, and cua-driver's default dispatch+auto-selects UIA-Invoke where an a11y tree exists and falls back to+pixel/pointer-injection where it doesn't.++## Layout (each window = Β½ work-width Γ Β½ work-height)++The four corners tile the taskbar-safe work area into quadrants; the master is+centered, **overlapping all four**:++```+ ββββββββββββββββββββββββββ¬βββββββββββββββββββββββββ+ β Win32 GDI (NO a11y) β WinForms (.NET) β+ β crimson β β amber β β+ β ββββββββββββββββββββββββββ β+ β β MASTER β Win32 controls β β you β+ βββββββββββββ (foreground, overlaps) βββββββββββββββ€+ β WPF (XAML)β β Electron β+ β ββββββββββββββββββββββββββ mint_lime β β+ β aqua β β (Chromium) β+ ββββββββββββββββββββββββββ΄βββββββββββββββββββββββββ+```++Click **SUBMIT** (or type a subject name then submit) in the center master:+four coloured cursors glide onto the four corner terminals and commit the same+record there β concurrently, in the background. Watch each corner's+green-screen records grid grow and its `RECORDS:` counter tick up, without any+corner ever coming to the front.++## Frameworks (and what they exercise)++| Window | Framework | Accessibility | cua-driver path |+|---|---|---|---|+| TL | Win32 + GDI (custom-drawn) | **none** | pixel hit-test β PostMessage / pointer injection |+| TR | .NET WinForms | MSAA/UIA | UIA Invoke |+| BL | .NET WPF | UIA (XAML) | UIA Invoke (no foreground steal via WS_EX_NOACTIVATE) |+| BR | Electron | UIA (Chromium) | UIA Invoke |+| Center | Win32 standard controls | MSAA | (foreground; the human drives it) |++## Build++```powershell+# from this directory+cargo build # legacy-app + orchestrator (Rust)+dotnet build dotnet/winforms/winforms.csproj # WinForms+dotnet build dotnet/wpf/wpf.csproj # WPF+npm install --prefix electron # Electron (downloads electron once)+```+Also build the driver once (repo root workspace):+```powershell+cargo build -p cua-driver --manifest-path ..\..\libs\cua-driver\rust\Cargo.toml+```++## Run++```powershell+.\target\debug\orchestrator.exe # human-driven: click/type in the center+.\target\debug\orchestrator.exe --auto # self-playing: drives a TYPE+CLICK every few seconds+```++The orchestrator starts the cua-driver daemon, launches + positions all five+windows, and fans every center action out to the four corners over four+concurrent `cua-driver call` sessions (`crimson` / `amber` / `aqua` /+`mint_lime` β four cursor colours). Close the center window (or kill the+orchestrator) to tear everything down β a Windows Job Object kills the whole+tree, so nothing is orphaned.++### Env overrides+`CUA_DRIVER_EXE`, `LEGACY_APP_EXE`, `WINFORMS_EXE`, `WPF_EXE`, `ELECTRON_DIR`.++## How the coloured cursors work+cua-driver assigns each session a cursor colour by name (palette-name sessions+like `crimson` pick that colour directly). Passing `"session":"<color>"` on each+`click`/`type_text` call routes it to that session's overlay cursor, which+glides to the target. Four sessions β four cursors animating at once. See+`docs/windows-background-input-re-plan.md` for the no-z-raise mechanism.diff --git a/demo/multi-cursor/dotnet/winforms/Program.cs b/demo/multi-cursor/dotnet/winforms/Program.csnew file mode 100644index 0000000000..88759c4c59--- /dev/null+++ b/demo/multi-cursor/dotnet/winforms/Program.cs@@ -0,0 +1,99 @@+using System;+using System.Collections.Generic;+using System.Drawing;+using System.Windows.Forms;++// "Meridian CRM β Account Record" : WinForms node (classic Win32 controls,+// MSAA/UIA). Explicit fractional layout (dense, no gaps beyond borders).+// cua-driver drives the Account Name field + the "Add Record" button.+static class Program+{+ [STAThread]+ static void Main()+ {+ Application.EnableVisualStyles();+ Application.SetCompatibleTextRenderingDefault(false);++ var f = new Form { Text = "Meridian CRM β Account Record [WinForms]", ClientSize = new Size(960, 600),+ StartPosition = FormStartPosition.Manual, Font = new Font("Segoe UI", 9f), BackColor = Color.FromArgb(240,240,240) };++ Label band(string t, Color bg, ContentAlignment a = ContentAlignment.MiddleLeft, FontStyle fs = FontStyle.Regular)+ => new Label { Text = t, BackColor = bg, TextAlign = a, Font = new Font("Segoe UI", 9f, fs) };++ var menu = band(" File Edit View Record Tools Help", Color.FromArgb(247,247,247));+ var tool = band(" New Open Save Delete β β Prev Next βΆ β Find", Color.FromArgb(235,235,235));+ var nameLbl = band(" Account Name", Color.FromArgb(240,240,240));+ var typeLbl = band(" Account Type", Color.FromArgb(240,240,240));+ var regionLbl = band(" Region", Color.FromArgb(240,240,240));+ var prioLbl = band(" Priority (1-5)", Color.FromArgb(240,240,240));+ var creditLbl = band(" Credit Limit", Color.FromArgb(240,240,240));+ var nameBox = new TextBox { BorderStyle = BorderStyle.FixedSingle };+ var typeCmb = new ComboBox { DropDownStyle = ComboBoxStyle.DropDownList, FlatStyle = FlatStyle.Flat };+ typeCmb.Items.AddRange(new object[] { "Enterprise", "SMB", "Government", "Reseller" }); typeCmb.SelectedIndex = 0;+ var regionCmb = new ComboBox { DropDownStyle = ComboBoxStyle.DropDownList, FlatStyle = FlatStyle.Flat };+ regionCmb.Items.AddRange(new object[] { "North", "South", "EMEA", "APAC", "LATAM" }); regionCmb.SelectedIndex = 0;+ var prio = new TrackBar { Minimum = 1, Maximum = 5, Value = 3, TickStyle = TickStyle.BottomRight };+ var credit = new TrackBar { Minimum = 0, Maximum = 100, Value = 40, TickFrequency = 10, TickStyle = TickStyle.BottomRight };+ var saveBtn = new Button { Text = "Add Record", FlatStyle = FlatStyle.System, Font = new Font("Segoe UI", 10f, FontStyle.Bold) };+ var sigHdr = band(" Signature / Notes", Color.FromArgb(225,225,225), ContentAlignment.MiddleLeft, FontStyle.Bold);+ var sig = new Panel { BackColor = Color.White, BorderStyle = BorderStyle.FixedSingle };+ var grid = new DataGridView { AllowUserToAddRows = false, ReadOnly = true, BackgroundColor = Color.White,+ BorderStyle = BorderStyle.FixedSingle, RowHeadersVisible = false, AllowUserToResizeRows = false,+ ColumnHeadersHeightSizeMode = DataGridViewColumnHeadersHeightSizeMode.DisableResizing };+ grid.Columns.Add("acct", "Account"); grid.Columns.Add("type", "Type"); grid.Columns.Add("region", "Region");+ grid.Columns.Add("pri", "Pri"); grid.Columns.Add("credit", "Credit");+ grid.Columns["acct"].AutoSizeMode = DataGridViewAutoSizeColumnMode.Fill;+ var status = band(" Ready", Color.FromArgb(232,232,232));+ var recLbl = band("Records: 0 USER: SYSTEM β CONNECTED ", Color.FromArgb(232,232,232), ContentAlignment.MiddleRight);++ // line tool: one straight segment per press-drag-release (downβup)+ var segs = new List<(Point a, Point b)>(); Point? down = null; Point cur = Point.Empty;+ sig.MouseDown += (s, e) => { down = e.Location; cur = e.Location; };+ sig.MouseMove += (s, e) => { if (down != null) { cur = e.Location; sig.Invalidate(); } };+ sig.MouseUp += (s, e) => { if (down != null) { segs.Add((down.Value, e.Location)); down = null; sig.Invalidate(); } };+ sig.Paint += (s, e) => { e.Graphics.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.AntiAlias;+ using var p = new Pen(Color.MidnightBlue, 2); foreach (var sg in segs) e.Graphics.DrawLine(p, sg.a, sg.b);+ if (down != null) { using var pp = new Pen(Color.FromArgb(110, 25, 25, 112), 1); e.Graphics.DrawLine(pp, down.Value, cur); } };++ int records = 0;+ saveBtn.Click += (s, e) =>+ {+ string nm = nameBox.Text.Trim(); if (nm.Length == 0) nm = "(unnamed)";+ grid.Rows.Add(nm, typeCmb.Text, regionCmb.Text, prio.Value, "$" + (credit.Value * 1000));+ if (grid.Rows.Count > 0) grid.FirstDisplayedScrollingRowIndex = grid.Rows.Count - 1;+ records++; status.Text = $" Saved account '{nm}'."; recLbl.Text = $"Records: {records} USER: SYSTEM β CONNECTED ";+ nameBox.Clear();+ };++ foreach (Control c in new Control[] { menu, tool, nameLbl, typeLbl, regionLbl, prioLbl, creditLbl,+ nameBox, typeCmb, regionCmb, prio, credit, saveBtn, sigHdr, sig, grid, status, recLbl })+ f.Controls.Add(c);++ void Layout()+ {+ int W = f.ClientSize.Width, H = f.ClientSize.Height;+ int X(double a) => (int)(W * a); int Y(double a) => (int)(H * a);+ Rectangle R(double x0, double y0, double x1, double y1) => Rectangle.FromLTRB(X(x0), Y(y0), X(x1), Y(y1));+ menu.Bounds = R(0, 0, 1, 0.045);+ tool.Bounds = R(0, 0.045, 1, 0.095);+ double lc = 0.13; // label/control split+ nameLbl.Bounds = R(0, 0.105, lc, 0.16); nameBox.Bounds = R(lc, 0.105, 0.42, 0.16);+ typeLbl.Bounds = R(0, 0.165, lc, 0.22); typeCmb.Bounds = R(lc, 0.167, 0.42, 0.22);+ regionLbl.Bounds = R(0, 0.225, lc, 0.28); regionCmb.Bounds = R(lc, 0.227, 0.42, 0.28);+ prioLbl.Bounds = R(0, 0.285, lc, 0.345); prio.Bounds = R(lc, 0.285, 0.42, 0.345);+ creditLbl.Bounds = R(0, 0.35, lc, 0.41); credit.Bounds = R(lc, 0.35, 0.42, 0.41);+ saveBtn.Bounds = R(0.04, 0.45, 0.40, 0.52);+ sigHdr.Bounds = R(0.42, 0.095, 1, 0.135);+ sig.Bounds = R(0.42, 0.135, 0.995, 0.42);+ grid.Bounds = R(0.005, 0.55, 0.995, 0.93);+ status.Bounds = R(0, 0.94, 0.5, 1);+ recLbl.Bounds = R(0.5, 0.94, 1, 1);+ float gfs = Math.Max(8, H / 70f);+ grid.Font = new Font("Segoe UI", gfs);+ grid.ColumnHeadersDefaultCellStyle.Font = new Font("Segoe UI", gfs, FontStyle.Bold);+ }+ f.Load += (s, e) => Layout();+ f.Resize += (s, e) => Layout();+ Application.Run(f);+ }+}diff --git a/demo/multi-cursor/dotnet/winforms/winforms.csproj b/demo/multi-cursor/dotnet/winforms/winforms.csprojnew file mode 100644index 0000000000..5b6864dd2d--- /dev/null+++ b/demo/multi-cursor/dotnet/winforms/winforms.csproj@@ -0,0 +1,11 @@+<Project Sdk="Microsoft.NET.Sdk">+ <PropertyGroup>+ <OutputType>WinExe</OutputType>+ <TargetFramework>net10.0-windows</TargetFramework>+ <UseWindowsForms>true</UseWindowsForms>+ <Nullable>disable</Nullable>+ <ImplicitUsings>disable</ImplicitUsings>+ <AssemblyName>winforms-legacy</AssemblyName>+ <ApplicationHighDpiMode>PerMonitorV2</ApplicationHighDpiMode>+ </PropertyGroup>+</Project>diff --git a/demo/multi-cursor/dotnet/wpf/Program.cs b/demo/multi-cursor/dotnet/wpf/Program.csnew file mode 100644index 0000000000..7a5f07348e--- /dev/null+++ b/demo/multi-cursor/dotnet/wpf/Program.cs@@ -0,0 +1,131 @@+using System;+using System.Collections.ObjectModel;+using System.Windows;+using System.Windows.Controls;+using System.Windows.Controls.Primitives;+using System.Windows.Media;+using System.Windows.Shapes;++// "Meridian CRM β Account Record" : WPF node (XAML / UIA). Real Slider /+// ComboBox / InkCanvas (doodle) / DataGrid (spreadsheet). Account Name TextBox+// + SAVE button are what cua-driver drives.+class Program+{+ public class RowVM+ {+ public string Account { get; set; } public string Type { get; set; }+ public string Region { get; set; } public int Pri { get; set; } public string Credit { get; set; }+ }++ [STAThread]+ static void Main()+ {+ var app = new Application();+ var rows = new ObservableCollection<RowVM>();+ var gray = new SolidColorBrush(Color.FromRgb(0xE0, 0xE0, 0xE0));++ // menu+ var menu = new Menu();+ foreach (var m in new[] { "File", "Edit", "View", "Record", "Tools", "Help" })+ menu.Items.Add(new MenuItem { Header = m });+ // toolbar+ var tray = new ToolBarTray();+ var tb = new ToolBar();+ foreach (var t in new[] { "New", "Open", "Save", "Delete", "|", "β Prev", "Next βΆ", "|", "Find" })+ tb.Items.Add(t == "|" ? (object)new Separator() : new Button { Content = t, Padding = new Thickness(6, 1, 6, 1) });+ tray.ToolBars.Add(tb);++ // ββ left form (dense grid; cell borders only) βββββββββββββββββββββββ+ var form = new Grid { Background = Brushes.White };+ form.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(120) });+ form.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(1, GridUnitType.Star) });+ TextBlock lbl(string t) => new TextBlock { Text = " " + t, VerticalAlignment = VerticalAlignment.Center, Background = new SolidColorBrush(Color.FromRgb(0xF0, 0xF0, 0xF0)) };+ int r = 0;+ void AddRow(string label, FrameworkElement c, double h = 30)+ {+ form.RowDefinitions.Add(new RowDefinition { Height = new GridLength(h) });+ var b1 = new Border { BorderBrush = gray, BorderThickness = new Thickness(0, 0, 1, 1), Child = lbl(label) };+ Grid.SetRow(b1, r); Grid.SetColumn(b1, 0); form.Children.Add(b1);+ var b2 = new Border { BorderBrush = gray, BorderThickness = new Thickness(0, 0, 0, 1), Child = c };+ Grid.SetRow(b2, r); Grid.SetColumn(b2, 1); form.Children.Add(b2);+ r++;+ }+ var nameBox = new TextBox { BorderThickness = new Thickness(0), VerticalContentAlignment = VerticalAlignment.Center };+ var typeCmb = new ComboBox { BorderThickness = new Thickness(0) };+ foreach (var s in new[] { "Enterprise", "SMB", "Government", "Reseller" }) typeCmb.Items.Add(s); typeCmb.SelectedIndex = 0;+ var regionCmb = new ComboBox { BorderThickness = new Thickness(0) };+ foreach (var s in new[] { "North", "South", "EMEA", "APAC", "LATAM" }) regionCmb.Items.Add(s); regionCmb.SelectedIndex = 0;+ var priority = new Slider { Minimum = 1, Maximum = 5, Value = 3, TickFrequency = 1, IsSnapToTickEnabled = true, TickPlacement = TickPlacement.BottomRight, VerticalAlignment = VerticalAlignment.Center };+ var credit = new Slider { Minimum = 0, Maximum = 100, Value = 40, TickFrequency = 10, TickPlacement = TickPlacement.BottomRight, VerticalAlignment = VerticalAlignment.Center };+ AddRow("Account Name", nameBox);+ AddRow("Account Type", typeCmb);+ AddRow("Region", regionCmb);+ AddRow("Priority (1-5)", priority);+ AddRow("Credit Limit", credit);+ var saveBtn = new Button { Content = "Add Record", FontWeight = FontWeights.Bold };+ form.RowDefinitions.Add(new RowDefinition { Height = new GridLength(40) });+ Grid.SetRow(saveBtn, r); Grid.SetColumn(saveBtn, 0); Grid.SetColumnSpan(saveBtn, 2); form.Children.Add(saveBtn);++ // ββ right top: line-tool sketch (one straight segment per drag) βββββ+ var ink = new Canvas { Background = Brushes.White, ClipToBounds = true };+ Point? dn = null; Line preview = null;+ ink.MouseLeftButtonDown += (s, e) => { dn = e.GetPosition(ink); };+ ink.MouseMove += (s, e) => { if (dn != null) { var p = e.GetPosition(ink);+ if (preview == null) { preview = new Line { Stroke = Brushes.MidnightBlue, StrokeThickness = 1, Opacity = 0.45 }; ink.Children.Add(preview); }+ preview.X1 = dn.Value.X; preview.Y1 = dn.Value.Y; preview.X2 = p.X; preview.Y2 = p.Y; } };+ ink.MouseLeftButtonUp += (s, e) => { if (dn != null) { var p = e.GetPosition(ink);+ ink.Children.Add(new Line { Stroke = Brushes.MidnightBlue, StrokeThickness = 2, X1 = dn.Value.X, Y1 = dn.Value.Y, X2 = p.X, Y2 = p.Y });+ if (preview != null) { ink.Children.Remove(preview); preview = null; }+ dn = null; } };+ var sigHdr = new TextBlock { Text = " Signature / Notes", FontWeight = FontWeights.Bold, Background = new SolidColorBrush(Color.FromRgb(0xE1,0xE1,0xE1)), Padding = new Thickness(2) };+ var sigDock = new DockPanel(); DockPanel.SetDock(sigHdr, Dock.Top); sigDock.Children.Add(sigHdr); sigDock.Children.Add(ink);++ // ββ right bottom: spreadsheet βββββββββββββββββββββββββββββββββββββββ+ var dg = new DataGrid { AutoGenerateColumns = false, ItemsSource = rows, IsReadOnly = true, GridLinesVisibility = DataGridGridLinesVisibility.All, HeadersVisibility = DataGridHeadersVisibility.Column };+ dg.Columns.Add(new DataGridTextColumn { Header = "Account", Binding = new System.Windows.Data.Binding("Account"), Width = new DataGridLength(1, DataGridLengthUnitType.Star) });+ dg.Columns.Add(new DataGridTextColumn { Header = "Type", Binding = new System.Windows.Data.Binding("Type") });+ dg.Columns.Add(new DataGridTextColumn { Header = "Region", Binding = new System.Windows.Data.Binding("Region") });+ dg.Columns.Add(new DataGridTextColumn { Header = "Pri", Binding = new System.Windows.Data.Binding("Pri") });+ dg.Columns.Add(new DataGridTextColumn { Header = "Credit", Binding = new System.Windows.Data.Binding("Credit") });++ var rightGrid = new Grid();+ rightGrid.RowDefinitions.Add(new RowDefinition { Height = new GridLength(150) });+ rightGrid.RowDefinitions.Add(new RowDefinition { Height = new GridLength(3) });+ rightGrid.RowDefinitions.Add(new RowDefinition { Height = new GridLength(1, GridUnitType.Star) });+ Grid.SetRow(sigDock, 0); rightGrid.Children.Add(sigDock);+ var gs2 = new GridSplitter { Height = 3, HorizontalAlignment = HorizontalAlignment.Stretch, Background = gray }; Grid.SetRow(gs2, 1); rightGrid.Children.Add(gs2);+ Grid.SetRow(dg, 2); rightGrid.Children.Add(dg);++ var main = new Grid();+ main.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(380) });+ main.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(3) });+ main.ColumnDefinitions.Add(new ColumnDefinition { Width = new GridLength(1, GridUnitType.Star) });+ Grid.SetColumn(form, 0); main.Children.Add(form);+ var gs1 = new GridSplitter { Width = 3, Background = gray }; Grid.SetColumn(gs1, 1); main.Children.Add(gs1);+ Grid.SetColumn(rightGrid, 2); main.Children.Add(rightGrid);++ // status bar+ var statusBar = new StatusBar();+ var statusLbl = new StatusBarItem { Content = "Ready" };+ var recLbl = new StatusBarItem { Content = "Records: 0", HorizontalAlignment = HorizontalAlignment.Right };+ statusBar.Items.Add(statusLbl);+ statusBar.Items.Add(new StatusBarItem { Content = "USER: SYSTEM β CONNECTED", HorizontalAlignment = HorizontalAlignment.Right });+ statusBar.Items.Add(recLbl);++ int records = 0;+ saveBtn.Click += (s, e) =>+ {+ string nm = (nameBox.Text ?? "").Trim(); if (nm.Length == 0) nm = "(unnamed)";+ rows.Add(new RowVM { Account = nm, Type = typeCmb.Text, Region = regionCmb.Text, Pri = (int)priority.Value, Credit = "$" + ((int)credit.Value * 1000) });+ records++; recLbl.Content = $"Records: {records}"; statusLbl.Content = $"Saved account '{nm}'.";+ nameBox.Text = "";+ };++ var dock = new DockPanel();+ DockPanel.SetDock(menu, Dock.Top); DockPanel.SetDock(tray, Dock.Top); DockPanel.SetDock(statusBar, Dock.Bottom);+ dock.Children.Add(menu); dock.Children.Add(tray); dock.Children.Add(statusBar); dock.Children.Add(main);++ var win = new Window { Title = "Meridian CRM β Account Record [WPF]", Width = 960, Height = 600, Content = dock, WindowStartupLocation = WindowStartupLocation.Manual };+ app.Run(win);+ }+}diff --git a/demo/multi-cursor/dotnet/wpf/wpf.csproj b/demo/multi-cursor/dotnet/wpf/wpf.csprojnew file mode 100644index 0000000000..b2b65b75df--- /dev/null+++ b/demo/multi-cursor/dotnet/wpf/wpf.csproj@@ -0,0 +1,11 @@+<Project Sdk="Microsoft.NET.Sdk">+ <PropertyGroup>+ <OutputType>WinExe</OutputType>+ <TargetFramework>net10.0-windows</TargetFramework>+ <UseWPF>true</UseWPF>+ <Nullable>disable</Nullable>+ <ImplicitUsings>disable</ImplicitUsings>+ <AssemblyName>wpf-legacy</AssemblyName>+ <EnableDefaultApplicationDefinition>false</EnableDefaultApplicationDefinition>+ </PropertyGroup>+</Project>diff --git a/demo/multi-cursor/electron/index.html b/demo/multi-cursor/electron/index.htmlnew file mode 100644index 0000000000..1cb72ad276--- /dev/null+++ b/demo/multi-cursor/electron/index.html@@ -0,0 +1,103 @@+<!doctype html>+<html>+<head>+<meta charset="utf-8" />+<style>+ * { box-sizing: border-box; margin: 0; padding: 0; }+ html, body { height: 100%; font-family: "Segoe UI", Tahoma, sans-serif; font-size: 13px; background: #f0f0f0; overflow: hidden; }+ #app { display: grid; grid-template-rows: auto auto 1fr auto; height: 100vh; }+ /* menu + toolbar */+ #menu { background: #f7f7f7; border-bottom: 1px solid #c8c8c8; padding: 2px 0; }+ #menu span { padding: 3px 10px; }+ #menu span:hover { background: #cde2ff; }+ #toolbar { background: #eee; border-bottom: 1px solid #c8c8c8; display: flex; }+ #toolbar button { border: 1px solid transparent; background: transparent; padding: 3px 9px; }+ #toolbar button:hover { border: 1px solid #b0c4de; background: #e6eefb; }+ #toolbar .sep { width: 1px; background: #c8c8c8; margin: 2px 3px; }+ /* main split */+ #main { display: grid; grid-template-columns: 380px 1px 1fr; overflow: hidden; }+ #split { background: #c8c8c8; }+ /* dense form: label | control rows, only cell borders */+ #form { display: grid; grid-template-columns: 120px 1fr; grid-auto-rows: 30px; background: #fff; align-content: start; }+ #form .lbl { background: #f0f0f0; border-right: 1px solid #e0e0e0; border-bottom: 1px solid #e0e0e0; display: flex; align-items: center; padding-left: 8px; }+ #form .cell { border-bottom: 1px solid #e0e0e0; display: flex; align-items: center; }+ #form .cell > * { width: 100%; height: 100%; border: 0; background: transparent; padding: 0 6px; font: inherit; }+ #form input[type=range] { padding: 0; }+ #saveRow { grid-column: 1 / span 2; }+ #save { width: 100%; height: 38px; font-weight: bold; }+ /* right side */+ #right { display: grid; grid-template-rows: 150px 1px 1fr; overflow: hidden; }+ #sigHdr { background: #e1e1e1; font-weight: bold; padding: 3px 6px; border-bottom: 1px solid #c8c8c8; }+ #sigWrap { display: grid; grid-template-rows: auto 1fr; }+ #doodle { width: 100%; height: 100%; background: #fff; display: block; }+ table { border-collapse: collapse; width: 100%; background: #fff; }+ th, td { border: 1px solid #d2d2d2; padding: 2px 6px; text-align: left; font-weight: normal; }+ th { background: #eef1f5; font-weight: 600; }+ #gridWrap { overflow: auto; }+ #status { background: #e8e8e8; border-top: 1px solid #c8c8c8; display: flex; justify-content: space-between; padding: 3px 8px; }+</style>+</head>+<body>+ <div id="app">+ <div id="menu"><span>File</span><span>Edit</span><span>View</span><span>Record</span><span>Tools</span><span>Help</span></div>+ <div id="toolbar">+ <button>New</button><button>Open</button><button>Save</button><button>Delete</button>+ <div class="sep"></div><button>β Prev</button><button>Next βΆ</button>+ <div class="sep"></div><button>Find</button>+ </div>+ <div id="main">+ <div id="form">+ <div class="lbl">Account Name</div><div class="cell"><input id="acct" type="text" /></div>+ <div class="lbl">Account Type</div><div class="cell"><select id="type"><option>Enterprise</option><option>SMB</option><option>Government</option><option>Reseller</option></select></div>+ <div class="lbl">Region</div><div class="cell"><select id="region"><option>North</option><option>South</option><option>EMEA</option><option>APAC</option><option>LATAM</option></select></div>+ <div class="lbl">Priority (1-5)</div><div class="cell"><input id="pri" type="range" min="1" max="5" value="3" /></div>+ <div class="lbl">Credit Limit</div><div class="cell"><input id="credit" type="range" min="0" max="100" value="40" /></div>+ <div id="saveRow" class="cell"><button id="save">Add Record</button></div>+ </div>+ <div id="split"></div>+ <div id="right">+ <div id="sigWrap"><div id="sigHdr">Signature / Notes</div><canvas id="doodle"></canvas></div>+ <div id="split"></div>+ <div id="gridWrap">+ <table id="grid"><thead><tr><th>Account</th><th>Type</th><th>Region</th><th>Pri</th><th>Credit</th></tr></thead><tbody></tbody></table>+ </div>+ </div>+ </div>+ <div id="status"><span id="statusMsg">Ready</span><span>USER: SYSTEM β CONNECTED</span><span id="recCount">Records: 0</span></div>+ </div>+ <script>+ // line tool: one straight segment per press-drag-release (downβup)+ const cv = document.getElementById('doodle');+ const ctx = cv.getContext('2d');+ let segs = [], dn = null, cur = null;+ function redraw() {+ ctx.clearRect(0, 0, cv.width, cv.height);+ ctx.strokeStyle = '#191970'; ctx.lineWidth = 2;+ for (const s of segs) { ctx.beginPath(); ctx.moveTo(s.x0, s.y0); ctx.lineTo(s.x1, s.y1); ctx.stroke(); }+ if (dn && cur) { ctx.save(); ctx.globalAlpha = 0.45; ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(dn.x, dn.y); ctx.lineTo(cur.x, cur.y); ctx.stroke(); ctx.restore(); }+ }+ function fit() { cv.width = cv.clientWidth; cv.height = cv.clientHeight; redraw(); }+ window.addEventListener('resize', fit); requestAnimationFrame(fit);+ // Pointer events (not mouse): coordinate-routed background injection arrives+ // as PEN/touch pointers; pointer* covers mouse + pen + touch uniformly.+ cv.style.touchAction = 'none';+ cv.addEventListener('pointerdown', e => { dn = { x: e.offsetX, y: e.offsetY }; cur = dn; });+ cv.addEventListener('pointermove', e => { if (dn) { cur = { x: e.offsetX, y: e.offsetY }; redraw(); } });+ cv.addEventListener('pointerup', e => { if (dn) { segs.push({ x0: dn.x, y0: dn.y, x1: e.offsetX, y1: e.offsetY }); dn = null; cur = null; redraw(); } });+ // save+ let records = 0;+ const acct = document.getElementById('acct'), tbody = document.querySelector('#grid tbody');+ document.getElementById('save').addEventListener('click', () => {+ let nm = (acct.value || '').trim(); if (!nm) nm = '(unnamed)';+ const tr = document.createElement('tr');+ const credit = '$' + (document.getElementById('credit').value * 1000);+ [nm, document.getElementById('type').value, document.getElementById('region').value, document.getElementById('pri').value, credit]+ .forEach(v => { const td = document.createElement('td'); td.textContent = v; tr.appendChild(td); });+ tbody.appendChild(tr);+ records++; document.getElementById('recCount').textContent = `Records: ${records}`;+ document.getElementById('statusMsg').textContent = `Saved account '${nm}'.`;+ acct.value = '';+ });+ </script>+</body>+</html>diff --git a/demo/multi-cursor/electron/main.js b/demo/multi-cursor/electron/main.jsnew file mode 100644index 0000000000..f8ef8da10e--- /dev/null+++ b/demo/multi-cursor/electron/main.js@@ -0,0 +1,24 @@+const { app, BrowserWindow } = require('electron');+const path = require('path');++function createWindow() {+ const win = new BrowserWindow({+ width: 900,+ height: 560,+ resizable: true,+ title: 'Meridian CRM β Account Record [Electron]',+ autoHideMenuBar: true,+ webPreferences: { contextIsolation: true },+ });+ win.setMenuBarVisibility(false);+ win.loadFile(path.join(__dirname, 'index.html'));+}++app.whenReady().then(() => {+ createWindow();+ app.on('activate', () => {+ if (BrowserWindow.getAllWindows().length === 0) createWindow();+ });+});++app.on('window-all-closed', () => app.quit());diff --git a/demo/multi-cursor/electron/package.json b/demo/multi-cursor/electron/package.jsonnew file mode 100644index 0000000000..0c36621cbf--- /dev/null+++ b/demo/multi-cursor/electron/package.json@@ -0,0 +1,12 @@+{+ "name": "cua-demo-electron-legacy",+ "version": "0.1.0",+ "private": true,+ "main": "main.js",+ "scripts": {+ "start": "electron ."+ },+ "devDependencies": {+ "electron": "^33.0.0"+ }+}diff --git a/demo/multi-cursor/legacy-app/Cargo.toml b/demo/multi-cursor/legacy-app/Cargo.tomlnew file mode 100644index 0000000000..59453e0f8d--- /dev/null+++ b/demo/multi-cursor/legacy-app/Cargo.toml@@ -0,0 +1,16 @@+[package]+name = "legacy-app"+version = "0.1.0"+edition = "2021"++[[bin]]+name = "legacy-app"+path = "src/main.rs"++[dependencies]+windows = { version = "0.58", features = [+ "Win32_Foundation",+ "Win32_UI_WindowsAndMessaging",+ "Win32_Graphics_Gdi",+ "Win32_System_LibraryLoader",+] }diff --git a/demo/multi-cursor/legacy-app/src/main.rs b/demo/multi-cursor/legacy-app/src/main.rsnew file mode 100644index 0000000000..8960a97458--- /dev/null+++ b/demo/multi-cursor/legacy-app/src/main.rs@@ -0,0 +1,343 @@+//! "Meridian CRM β Account Record" β a dense, realistic Windows office form for+//! the cua-driver multi-cursor demo. Menu bar, toolbar, a packed left form+//! (Account Name, Account Type / Region dropdowns, Priority / Credit sliders),+//! a SAVE button, a signature doodle pad, a spreadsheet grid, and a status bar.+//! No padding between elements beyond their 1px borders.+//!+//! Modes (argv[1]):+//! gdi <node> β fully GDI-drawn (NO accessibility tree β cua-driver drives+//! it via the pixel path). The Account-Name field and SAVE+//! button are drawn; the doodle pad is live.+//! master <node> β same chrome, but Account Name + SAVE are real Win32+//! controls, instrumented to EMIT the user's action on stdout+//! (TYPE<text>, CLICK<rx><ry>) for the orchestrator to replay.++#![windows_subsystem = "windows"]++use std::cell::RefCell;+use std::io::Write;++use windows::core::{w, PCWSTR};+use windows::Win32::Foundation::{COLORREF, HINSTANCE, HWND, LPARAM, LRESULT, POINT, RECT, WPARAM};+use windows::Win32::Graphics::Gdi::{+ BeginPaint, CreateFontW, CreatePen, CreateSolidBrush, DeleteObject, DrawTextW, EndPaint, FillRect,+ FrameRect, InvalidateRect, LineTo, MoveToEx, SelectObject, SetBkMode, SetTextColor, DT_CENTER, DT_LEFT,+ DT_SINGLELINE, DT_VCENTER, GetStockObject, HBRUSH, HFONT, PAINTSTRUCT, PS_SOLID, TRANSPARENT, BLACK_BRUSH,+};+use windows::Win32::System::LibraryLoader::GetModuleHandleW;+use windows::Win32::UI::WindowsAndMessaging::*;++// ββ dense fractional layout (shared by gdi + master) ββββββββββββββββββββββββββ+const MENU_B: f64 = 0.05;+const TOOL_B: f64 = 0.10;+// left form rows+const NAME: [f64; 4] = [0.16, 0.110, 0.41, 0.170]; // Account Name field box+const TYPE: [f64; 4] = [0.16, 0.175, 0.41, 0.235];+const REGION: [f64; 4] = [0.16, 0.240, 0.41, 0.300];+const PRIO: [f64; 4] = [0.16, 0.305, 0.41, 0.365];+const CREDIT: [f64; 4] = [0.16, 0.370, 0.41, 0.430];+const SAVE: [f64; 4] = [0.04, 0.460, 0.40, 0.530]; // SAVE button+const DOODLE: [f64; 4] = [0.42, 0.140, 0.99, 0.420];+const GRID: [f64; 4] = [0.01, 0.560, 0.99, 0.930];+const STATUS_T: f64 = 0.94;+// SAVE button relative center (master emits this for the CLICK event).+const SAVE_CX: f64 = (SAVE[0] + SAVE[2]) / 2.0;+const SAVE_CY: f64 = (SAVE[1] + SAVE[3]) / 2.0;++const ID_EDIT: isize = 1001;+const ID_BUTTON: isize = 1002;++#[derive(Clone, Copy, PartialEq)]+enum Mode { Gdi, Master }++struct State {+ mode: Mode,+ node: String,+ records: Vec<[String; 5]>,+ name: String,+ // line tool: one straight segment per press-drag-release (downβup)+ segs: Vec<(POINT, POINT)>,+ down: Option<POINT>,+ cur: POINT,+ hedit: HWND,+ hbtn: HWND,+ f_title: HFONT,+ f_label: HFONT,+ f_mono: HFONT,+ cw: i32,+ ch: i32,+}++thread_local! { static STATE: RefCell<Option<State>> = const { RefCell::new(None) }; }++fn emit(line: &str) { let _ = writeln!(std::io::stdout(), "{line}"); let _ = std::io::stdout().flush(); }+fn wide(s: &str) -> Vec<u16> { s.encode_utf16().chain(std::iter::once(0)).collect() }+fn fr(cw: i32, ch: i32, r: [f64; 4]) -> RECT {+ RECT { left: (cw as f64 * r[0]) as i32, top: (ch as f64 * r[1]) as i32,+ right: (cw as f64 * r[2]) as i32, bottom: (ch as f64 * r[3]) as i32 }+}+fn band(cw: i32, ch: i32, y0: f64, y1: f64) -> RECT {+ RECT { left: 0, top: (ch as f64 * y0) as i32, right: cw, bottom: (ch as f64 * y1) as i32 }+}+fn pt_in(r: &RECT, x: i32, y: i32) -> bool { x >= r.left && x < r.right && y >= r.top && y < r.bottom }+unsafe fn mk_font(h: i32, w: i32, face: PCWSTR) -> HFONT { CreateFontW(h, 0, 0, 0, w, 0, 0, 0, 0, 0, 0, 0, 0, face) }++fn main() {+ let args: Vec<String> = std::env::args().collect();+ let mode = if args.get(1).map(|s| s.as_str()) == Some("master") { Mode::Master } else { Mode::Gdi };+ let node = args.get(2).cloned().unwrap_or_else(|| match mode { Mode::Master => "Master".into(), Mode::Gdi => "Win32 GDI".into() });++ unsafe {+ let hmod = GetModuleHandleW(None).unwrap();+ let class = w!("MeridianCrmForm");+ let bg = CreateSolidBrush(COLORREF(0x00F0F0F0));+ let wc = WNDCLASSW { lpfnWndProc: Some(wnd_proc), hInstance: hmod.into(), lpszClassName: class,+ hbrBackground: bg, hCursor: LoadCursorW(None, IDC_ARROW).unwrap_or_default(), ..Default::default() };+ RegisterClassW(&wc);++ STATE.with(|s| *s.borrow_mut() = Some(State { mode, node: node.clone(), records: Vec::new(),+ name: String::new(), segs: Vec::new(), down: None, cur: POINT { x: 0, y: 0 }, hedit: HWND::default(), hbtn: HWND::default(),+ f_title: HFONT::default(), f_label: HFONT::default(), f_mono: HFONT::default(), cw: 0, ch: 0 }));++ let title = format!("Meridian CRM β Account Record [{node}]");+ let tw = wide(&title);+ let hwnd = CreateWindowExW(WINDOW_EX_STYLE(0), class, PCWSTR(tw.as_ptr()), WS_OVERLAPPEDWINDOW,+ CW_USEDEFAULT, CW_USEDEFAULT, 960, 600, None, None, HINSTANCE(hmod.0), None).expect("CreateWindowExW");+ let _ = ShowWindow(hwnd, SW_SHOWNORMAL);++ let mut msg = MSG::default();+ while GetMessageW(&mut msg, None, 0, 0).as_bool() { let _ = TranslateMessage(&msg); DispatchMessageW(&msg); }+ }+}++unsafe fn create_master_controls(parent: HWND) {+ let hinst = HINSTANCE(GetModuleHandleW(None).unwrap().0);+ let hedit = CreateWindowExW(WS_EX_CLIENTEDGE, w!("EDIT"), w!(""),+ WS_CHILD | WS_VISIBLE | WINDOW_STYLE(ES_AUTOHSCROLL as u32),+ 0, 0, 10, 10, parent, HMENU(ID_EDIT as *mut core::ffi::c_void), hinst, None).unwrap_or_default();+ let hbtn = CreateWindowExW(WINDOW_EX_STYLE(0), w!("BUTTON"), w!("Add Record"),+ WS_CHILD | WS_VISIBLE | WINDOW_STYLE(BS_PUSHBUTTON as u32),+ 0, 0, 10, 10, parent, HMENU(ID_BUTTON as *mut core::ffi::c_void), hinst, None).unwrap_or_default();+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() { st.hedit = hedit; st.hbtn = hbtn; });+}++unsafe fn relayout(hwnd: HWND, cw: i32, ch: i32) {+ STATE.with(|s| {+ let mut b = s.borrow_mut(); let Some(st) = b.as_mut() else { return };+ st.cw = cw; st.ch = ch;+ for f in [st.f_title, st.f_label, st.f_mono] { if !f.is_invalid() { let _ = DeleteObject(f); } }+ let seg = wide("Segoe UI"); let mono = wide("Consolas");+ st.f_title = mk_font(-(ch / 34).clamp(12, 22), 700, PCWSTR(seg.as_ptr()));+ st.f_label = mk_font(-(ch / 40).clamp(11, 18), 400, PCWSTR(seg.as_ptr()));+ st.f_mono = mk_font(-(ch / 44).clamp(10, 16), 400, PCWSTR(mono.as_ptr()));+ if st.mode == Mode::Master && !st.hedit.0.is_null() {+ let n = fr(cw, ch, NAME);+ let _ = MoveWindow(st.hedit, n.left + 2, n.top + 2, n.right - n.left - 4, n.bottom - n.top - 4, true);+ let sv = fr(cw, ch, SAVE);+ let _ = MoveWindow(st.hbtn, sv.left, sv.top, sv.right - sv.left, sv.bottom - sv.top, true);+ SendMessageW(st.hedit, WM_SETFONT, WPARAM(st.f_label.0 as usize), LPARAM(1));+ SendMessageW(st.hbtn, WM_SETFONT, WPARAM(st.f_label.0 as usize), LPARAM(1));+ }+ });+ let _ = InvalidateRect(hwnd, None, true);+}++extern "system" fn wnd_proc(hwnd: HWND, msg: u32, wp: WPARAM, lp: LPARAM) -> LRESULT {+ unsafe {+ match msg {+ WM_CREATE => { if STATE.with(|s| s.borrow().as_ref().map(|st| st.mode)) == Some(Mode::Master) { create_master_controls(hwnd); } LRESULT(0) }+ WM_SIZE => { let (cw, ch) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32); if cw > 0 && ch > 0 { relayout(hwnd, cw, ch); } LRESULT(0) }+ WM_COMMAND => { if (wp.0 & 0xFFFF) as isize == ID_BUTTON && ((wp.0 >> 16) & 0xFFFF) as u32 == BN_CLICKED { on_submit(hwnd); } LRESULT(0) }+ WM_LBUTTONDOWN => {+ let (x, y) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32);+ let (mode, cw, ch) = STATE.with(|s| { let b = s.borrow(); let st = b.as_ref().unwrap(); (st.mode, st.cw, st.ch) });+ if mode == Mode::Gdi && pt_in(&fr(cw, ch, SAVE), x, y) { commit(hwnd); }+ else if pt_in(&fr(cw, ch, DOODLE), x, y) {+ // line tool: remember the press point; commit on button-up.+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() { st.down = Some(POINT { x, y }); st.cur = POINT { x, y }; });+ }+ LRESULT(0)+ }+ WM_MOUSEMOVE => {+ if (wp.0 & 0x0001) != 0 {+ let (x, y) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32);+ let go = STATE.with(|s| { let mut b = s.borrow_mut(); let st = b.as_mut().unwrap();+ if st.down.is_some() { st.cur = POINT { x, y }; true } else { false } });+ if go { let (cw, ch) = STATE.with(|s| { let b = s.borrow(); (b.as_ref().unwrap().cw, b.as_ref().unwrap().ch) });+ let d = fr(cw, ch, DOODLE); let _ = InvalidateRect(hwnd, Some(&d), false); }+ }+ LRESULT(0)+ }+ WM_LBUTTONUP => {+ let (x, y) = ((lp.0 & 0xFFFF) as i16 as i32, ((lp.0 >> 16) & 0xFFFF) as i16 as i32);+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() {+ if let Some(p0) = st.down.take() { st.segs.push((p0, POINT { x, y })); } });+ let (cw, ch) = STATE.with(|s| { let b = s.borrow(); (b.as_ref().unwrap().cw, b.as_ref().unwrap().ch) });+ let d = fr(cw, ch, DOODLE); let _ = InvalidateRect(hwnd, Some(&d), false);+ LRESULT(0)+ }+ WM_CHAR => {+ if STATE.with(|s| s.borrow().as_ref().map(|st| st.mode)) == Some(Mode::Gdi) {+ let ch = char::from_u32(wp.0 as u32).unwrap_or('\0');+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() {+ match ch { '\u{8}' => { st.name.pop(); }, '\r' | '\n' => {}, c if !c.is_control() => st.name.push(c), _ => {} } });+ let (cw, ch2) = STATE.with(|s| { let b = s.borrow(); (b.as_ref().unwrap().cw, b.as_ref().unwrap().ch) });+ let n = fr(cw, ch2, NAME); let _ = InvalidateRect(hwnd, Some(&n), false);+ }+ LRESULT(0)+ }+ WM_PAINT => { paint(hwnd); LRESULT(0) }+ WM_DESTROY => { PostQuitMessage(0); LRESULT(0) }+ _ => DefWindowProcW(hwnd, msg, wp, lp),+ }+ }+}++unsafe fn on_submit(hwnd: HWND) {+ let text = STATE.with(|s| { let b = s.borrow(); let st = b.as_ref()?; if st.hedit.0.is_null() { return None; }+ let len = GetWindowTextLengthW(st.hedit); let mut buf = vec![0u16; (len + 1) as usize];+ let n = GetWindowTextW(st.hedit, &mut buf); Some(String::from_utf16_lossy(&buf[..n as usize])) });+ if let Some(t) = text {+ emit(&format!("TYPE\t{t}"));+ emit(&format!("CLICK\t{SAVE_CX:.4}\t{SAVE_CY:.4}"));+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() { st.name = t; });+ commit(hwnd);+ STATE.with(|s| if let Some(st) = s.borrow().as_ref() { if !st.hedit.0.is_null() { let _ = SetWindowTextW(st.hedit, w!("")); } });+ }+}++unsafe fn commit(hwnd: HWND) {+ STATE.with(|s| if let Some(st) = s.borrow_mut().as_mut() {+ let nm = if st.name.trim().is_empty() { "(unnamed)".into() } else { st.name.trim().chars().take(22).collect::<String>() };+ st.records.push([nm, "Enterprise".into(), "North".into(), "3".into(), "$40,000".into()]);+ st.name.clear();+ });+ let _ = InvalidateRect(hwnd, None, true);+}++unsafe fn line(hdc: windows::Win32::Graphics::Gdi::HDC, x: i32, y: i32, text: &str) {+ let mut r = RECT { left: x, top: y, right: x + 6000, bottom: y + 60 }; let mut t = wide(text);+ DrawTextW(hdc, &mut t, &mut r, DT_LEFT | DT_SINGLELINE);+}++unsafe fn paint(hwnd: HWND) {+ let mut ps = PAINTSTRUCT::default();+ let hdc = BeginPaint(hwnd, &mut ps);+ SetBkMode(hdc, TRANSPARENT);+ STATE.with(|s| {+ let b = s.borrow(); let Some(st) = b.as_ref() else { return };+ let (cw, ch) = (st.cw.max(1), st.ch.max(1));+ let face = CreateSolidBrush(COLORREF(0x00F0F0F0));+ let bar = CreateSolidBrush(COLORREF(0x00E8E8E8));+ let hdr = CreateSolidBrush(COLORREF(0x00F5F1EE));+ let white = CreateSolidBrush(COLORREF(0x00FFFFFF));+ let line_c = CreateSolidBrush(COLORREF(0x00D2D2D2));+ let labelbg = CreateSolidBrush(COLORREF(0x00F0F0F0));+ let black = GetStockObject(BLACK_BRUSH);+ let sel = CreateSolidBrush(COLORREF(0x00F5C28A)); // accent for slider thumb / save++ let frame = |r: &RECT| { FrameRect(hdc, r, HBRUSH(black.0)); };+ let hline = |y: i32| { let r = RECT { left: 0, top: y, right: cw, bottom: y + 1 }; FillRect(hdc, &r, line_c); };+ let vline = |x: i32, y0: i32, y1: i32| { let r = RECT { left: x, top: y0, right: x + 1, bottom: y1 }; FillRect(hdc, &r, line_c); };++ // menu bar+ let mut m = band(cw, ch, 0.0, MENU_B); FillRect(hdc, &m, bar);+ SelectObject(hdc, st.f_label); SetTextColor(hdc, COLORREF(0x00000000));+ let mut mt = wide(" File Edit View Record Tools Help");+ DrawTextW(hdc, &mut mt, &mut m, DT_LEFT | DT_VCENTER | DT_SINGLELINE);+ hline((ch as f64 * MENU_B) as i32);+ // toolbar+ let mut tb = band(cw, ch, MENU_B, TOOL_B); FillRect(hdc, &tb, face);+ let mut tt = wide(" New Open Save Delete β β Prev Next βΆ β Find");+ DrawTextW(hdc, &mut tt, &mut tb, DT_LEFT | DT_VCENTER | DT_SINGLELINE);+ hline((ch as f64 * TOOL_B) as i32);++ // ββ left form ββ+ let label = |r: [f64; 4], text: &str| {+ let lr = RECT { left: (cw as f64 * 0.01) as i32, top: (ch as f64 * r[1]) as i32,+ right: (cw as f64 * (r[0] - 0.005)) as i32, bottom: (ch as f64 * r[3]) as i32 };+ FillRect(hdc, &lr, labelbg);+ let mut t = wide(text);+ let mut tr = RECT { left: lr.left + 6, ..lr };+ DrawTextW(hdc, &mut t, &mut tr, DT_LEFT | DT_VCENTER | DT_SINGLELINE);+ };+ let dropdown = |r: [f64; 4], val: &str| {+ let bx = fr(cw, ch, r); FillRect(hdc, &bx, white); frame(&bx);+ let mut tr = bx; tr.left += 6; tr.right -= 24; let mut t = wide(val);+ DrawTextW(hdc, &mut t, &mut tr, DT_LEFT | DT_VCENTER | DT_SINGLELINE);+ let ar = RECT { left: bx.right - 20, top: bx.top, right: bx.right, bottom: bx.bottom };+ FillRect(hdc, &ar, bar); frame(&ar); let mut a = wide("βΌ");+ DrawTextW(hdc, &mut a, &mut ar.clone(), DT_CENTER | DT_VCENTER | DT_SINGLELINE);+ };+ let slider = |r: [f64; 4], frac: f64| {+ let bx = fr(cw, ch, r); let midy = (bx.top + bx.bottom) / 2;+ let track = RECT { left: bx.left + 4, top: midy - 1, right: bx.right - 4, bottom: midy + 1 }; FillRect(hdc, &track, line_c);+ let tx = bx.left + 4 + ((bx.right - bx.left - 8) as f64 * frac) as i32;+ let thumb = RECT { left: tx - 4, top: bx.top + 6, right: tx + 4, bottom: bx.bottom - 6 }; FillRect(hdc, &thumb, sel); frame(&thumb);+ };+ SetTextColor(hdc, COLORREF(0x00000000));+ label(NAME, "Account Name"); label(TYPE, "Account Type"); label(REGION, "Region");+ label(PRIO, "Priority"); label(CREDIT, "Credit Limit");++ if st.mode == Mode::Gdi {+ let n = fr(cw, ch, NAME); FillRect(hdc, &n, white); frame(&n);+ let mut tr = n; tr.left += 6; let mut nm = wide(&st.name);+ DrawTextW(hdc, &mut nm, &mut tr, DT_LEFT | DT_VCENTER | DT_SINGLELINE);+ let sv = fr(cw, ch, SAVE); FillRect(hdc, &sv, sel); frame(&sv);+ let mut bt = wide("Add Record"); SelectObject(hdc, st.f_title);+ DrawTextW(hdc, &mut bt, &mut sv.clone(), DT_CENTER | DT_VCENTER | DT_SINGLELINE);+ SelectObject(hdc, st.f_label);+ }+ // dropdowns + sliders are visual in both modes+ dropdown(TYPE, "Enterprise"); dropdown(REGION, "North");+ slider(PRIO, 0.5); slider(CREDIT, 0.4);++ // ββ signature doodle ββ+ let dh = RECT { left: (cw as f64 * DOODLE[0]) as i32, top: (ch as f64 * 0.105) as i32,+ right: (cw as f64 * DOODLE[2]) as i32, bottom: (ch as f64 * DOODLE[1]) as i32 };+ FillRect(hdc, &dh, hdr); let mut sh = wide(" Signature / Notes");+ SelectObject(hdc, st.f_title); DrawTextW(hdc, &mut sh, &mut dh.clone(), DT_LEFT | DT_VCENTER | DT_SINGLELINE);+ SelectObject(hdc, st.f_label);+ let dz = fr(cw, ch, DOODLE); FillRect(hdc, &dz, white); frame(&dz);+ // line-tool segments (blue, 2px) + thin rubber-band preview while drawing+ let pen = CreatePen(PS_SOLID, 2, COLORREF(0x00701919));+ let old_pen = SelectObject(hdc, pen);+ for (a, b) in &st.segs { let _ = MoveToEx(hdc, a.x, a.y, None); let _ = LineTo(hdc, b.x, b.y); }+ if let Some(a) = st.down { let _ = MoveToEx(hdc, a.x, a.y, None); let _ = LineTo(hdc, st.cur.x, st.cur.y); }+ SelectObject(hdc, old_pen); let _ = DeleteObject(pen);++ // ββ spreadsheet ββ+ let g = fr(cw, ch, GRID); FillRect(hdc, &g, white); frame(&g);+ let cols = [("Account", 0.45), ("Type", 0.62), ("Region", 0.76), ("Pri", 0.85), ("Credit", 0.99)];+ let cx = |fx: f64| (cw as f64 * (GRID[0] + (GRID[2] - GRID[0]) * ((fx - 0.0) / 1.0))) as i32; // placeholder+ let _ = cx;+ // column x positions in absolute (fractions are of full width but cols listed as cumulative within grid)+ let gx = |fx: f64| (cw as f64 * fx) as i32;+ let row_h = ((st.ch as f64 / 44.0).clamp(10.0, 16.0) * 1.7) as i32;+ // header+ let hrow = RECT { left: g.left, top: g.top, right: g.right, bottom: g.top + row_h }; FillRect(hdc, &hrow, hdr);+ SelectObject(hdc, st.f_label);+ let mut prev = g.left + 6;+ for (name, fx) in cols { let mut t = wide(name); let mut tr = RECT { left: prev, top: g.top, right: gx(fx), bottom: g.top + row_h };+ DrawTextW(hdc, &mut t, &mut tr, DT_LEFT | DT_VCENTER | DT_SINGLELINE); vline(gx(fx), g.top, g.bottom); prev = gx(fx) + 6; }+ hline(g.top + row_h);+ // rows+ SelectObject(hdc, st.f_mono);+ let mut y = g.top + row_h + 2;+ for rec in &st.records {+ if y + row_h > g.bottom { break; }+ let xs = [g.left + 6, gx(0.45) + 6, gx(0.62) + 6, gx(0.76) + 6, gx(0.85) + 6];+ for (i, val) in rec.iter().enumerate() { line(hdc, xs[i], y + 2, val); }+ y += row_h; hline(y);+ }++ // ββ status bar ββ+ let mut sb = band(cw, ch, STATUS_T, 1.0); FillRect(hdc, &sb, bar); hline((ch as f64 * STATUS_T) as i32);+ SelectObject(hdc, st.f_label); SetTextColor(hdc, COLORREF(0x00000000));+ let mut sbt = wide(&format!(" Ready β Records: {} β USER: SYSTEM β CONNECTED β [{}]", st.records.len(), st.node));+ DrawTextW(hdc, &mut sbt, &mut sb, DT_LEFT | DT_VCENTER | DT_SINGLELINE);++ for o in [face, bar, hdr, white, line_c, labelbg, sel] { let _ = DeleteObject(o); }+ });+ let _ = EndPaint(hwnd, &ps);+}diff --git a/demo/multi-cursor/orchestrator/Cargo.toml b/demo/multi-cursor/orchestrator/Cargo.tomlnew file mode 100644index 0000000000..2ee4227380--- /dev/null+++ b/demo/multi-cursor/orchestrator/Cargo.toml@@ -0,0 +1,21 @@+[package]+name = "orchestrator"+version = "0.1.0"+edition = "2021"++[[bin]]+name = "orchestrator"+path = "src/main.rs"++[dependencies]+# PNG decode + resize for the screenshot-based maze line-diff verification.+image = { version = "0.25", default-features = false, features = ["png"] }+windows = { version = "0.58", features = [+ "Win32_Foundation",+ "Win32_UI_WindowsAndMessaging",+ "Win32_Graphics_Gdi",+ "Win32_Graphics_Dwm",+ "Win32_System_Threading",+ "Win32_System_JobObjects",+ "Win32_Security",+] }diff --git a/demo/multi-cursor/orchestrator/src/main.rs b/demo/multi-cursor/orchestrator/src/main.rsnew file mode 100644index 0000000000..8c4cfd1209--- /dev/null+++ b/demo/multi-cursor/orchestrator/src/main.rs@@ -0,0 +1,567 @@+//! Multi-cursor background computer-use demo orchestrator.+//!+//! Launches the "National Records System" terminal in five UI frameworks: a 2x2+//! grid of background windows (each = half the work area) around one foreground+//! "master" in the middle that overlaps all four. When the human submits a+//! record in the master, the master emits the action on stdout; this+//! orchestrator replays it onto all four background corners concurrently, each+//! via its OWN cua-driver session (= its own uniquely-coloured agent cursor),+//! in the background β no window raised, the user's cursor never moved.+//!+//! Driving is element-based where an accessibility tree exists (set_value for+//! the SUBJECT-NAME field + UIA-Invoke for SUBMIT β no foreground steal, no+//! SendInput) and pixel-based for the GDI corner that has no a11y tree. Each+//! a11y corner is VERIFIED by reading the records grid back: if the typed+//! record didn't land, it's logged as FAIL.++use std::io::{BufRead, BufReader};+use std::path::PathBuf;+use std::process::{Child, Command, Stdio};+use std::sync::mpsc::{channel, Sender};+use std::thread;+use std::time::{Duration, Instant};++use windows::core::PWSTR;+use windows::Win32::Foundation::{BOOL, HANDLE, HWND, LPARAM, POINT, RECT, TRUE};+use windows::Win32::Graphics::Dwm::{DwmGetWindowAttribute, DWMWA_EXTENDED_FRAME_BOUNDS};+use windows::Win32::Graphics::Gdi::ClientToScreen;+use windows::Win32::System::JobObjects::{+ AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject,+ JobObjectExtendedLimitInformation, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,+ JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,+};+use windows::Win32::UI::WindowsAndMessaging::{+ EnumWindows, GetClientRect, GetWindowTextW, GetWindowThreadProcessId, IsWindowVisible,+ SetForegroundWindow, SetWindowPos, HWND_TOP, SWP_NOACTIVATE, SWP_NOZORDER, SWP_SHOWWINDOW,+};++const FIELD_FRAC: (f64, f64) = (0.28, 0.145); // Account-Name field (GDI pixel fallback)++// ββ maze (line tool) βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+// A spiral maze as straight segments in [0,1] of the drawing REGION. cua-driver+// draws each as one press-drag-release (no curves = "line tool only"). The same+// segments rasterized are the reference the screenshot is diffed against.+const MAZE: [(f64, f64, f64, f64); 7] = [+ (0.10, 0.12, 0.90, 0.12),+ (0.90, 0.12, 0.90, 0.88),+ (0.90, 0.88, 0.26, 0.88),+ (0.26, 0.88, 0.26, 0.40),+ (0.26, 0.40, 0.66, 0.40),+ (0.66, 0.40, 0.66, 0.64),+ (0.66, 0.64, 0.46, 0.64),+];+// Client-fraction rect of the window that sits INSIDE every framework's drawing+// pad (all four put the doodle top-right). cua-driver draws here and the+// verifier crops the same rect β so the rasterized reference aligns regardless+// of where each framework actually lays its canvas out. Tuned via screenshots.+// Kept inside the SHORTEST drawing pad: WPF/Electron host theirs in a fixed+// ~150px row, so the usable band ends well above the spreadsheet below it.+// Overshooting here drops the lower maze onto the grid (and selects rows).+const REGION: [f64; 4] = [0.470, 0.180, 0.900, 0.295]; // l, t, r, b (client fractions)+// A drawn maze must score at least this (F1 of ink overlap vs the reference,+// dilated) to count as COMMITTED; below it = FAIL.+const MAZE_PASS: f64 = 0.45;++// ββ kill-on-exit job ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ+static JOB: std::sync::OnceLock<usize> = std::sync::OnceLock::new();+fn job() -> HANDLE {+ let raw = *JOB.get_or_init(|| unsafe {+ let h = CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW");+ let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();+ info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;+ let _ = SetInformationJobObject(h, JobObjectExtendedLimitInformation,+ &info as *const _ as *const core::ffi::c_void,+ std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32);+ h.0 as usize+ });+ HANDLE(raw as *mut core::ffi::c_void)+}+fn assign_to_job(child: &Child) {+ use std::os::windows::io::AsRawHandle;+ unsafe { let _ = AssignProcessToJobObject(job(), HANDLE(child.as_raw_handle() as *mut core::ffi::c_void)); }+}++struct Corner {+ title: &'static str, // unique substring to find the window+ session: &'static str, // palette name -> cursor color+ hwnd: HWND,+ pid: u32,+ field_idx: Option<u64>, // UIA element for the SUBJECT-NAME field (a11y corners)+ submit_idx: Option<u64>,// UIA element for the SUBMIT button+}++fn main() {+ let demo_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf();+ let repo_root = demo_root.parent().unwrap().parent().unwrap().to_path_buf();++ let cua = std::env::var("CUA_DRIVER_EXE").map(PathBuf::from)+ .unwrap_or_else(|_| repo_root.join("libs/cua-driver/rust/target/debug/cua-driver.exe"));+ let legacy_app = std::env::var("LEGACY_APP_EXE").map(PathBuf::from)+ .unwrap_or_else(|_| demo_root.join("target/debug/legacy-app.exe"));+ if !cua.exists() { eprintln!("cua-driver.exe not found at {cua:?}"); std::process::exit(1); }+ if !legacy_app.exists() { eprintln!("legacy-app.exe not found at {legacy_app:?}"); std::process::exit(1); }++ eprintln!("[orch] starting cua-driver daemonβ¦");+ let mut daemon = Command::new(&cua).arg("serve").stdout(Stdio::null()).stderr(Stdio::null())+ .spawn().expect("spawn cua-driver serve");+ assign_to_job(&daemon);+ thread::sleep(Duration::from_millis(1500));++ // Launch corners + master.+ let mut kids: Vec<Child> = Vec::new();+ let mut launch = |kids: &mut Vec<Child>, cmd: &mut Command| {+ if let Ok(c) = cmd.stdout(Stdio::null()).stderr(Stdio::null()).spawn() { assign_to_job(&c); kids.push(c); }+ };+ launch(&mut kids, Command::new(&legacy_app).args(["gdi", "Win32 GDI (no a11y)"]));+ let winforms = demo_root.join("dotnet/winforms/bin/Debug/net10.0-windows/winforms-legacy.exe");+ if winforms.exists() { launch(&mut kids, &mut Command::new(&winforms)); } else { eprintln!("[orch] (skip) winforms missing"); }+ let wpf = demo_root.join("dotnet/wpf/bin/Debug/net10.0-windows/wpf-legacy.exe");+ if wpf.exists() { launch(&mut kids, &mut Command::new(&wpf)); } else { eprintln!("[orch] (skip) wpf missing"); }+ let electron_bin = demo_root.join("electron/node_modules/.bin/electron.cmd");+ if electron_bin.exists() {+ let mut c = Command::new(&electron_bin); c.arg(".").current_dir(demo_root.join("electron"));+ launch(&mut kids, &mut c);+ } else { eprintln!("[orch] (skip) electron not installed"); }++ let mut master = Command::new(&legacy_app).args(["master", "Master (Win32 controls)"])+ .stdout(Stdio::piped()).stderr(Stdio::null()).spawn().expect("spawn master");+ assign_to_job(&master);+ let master_out = master.stdout.take().unwrap();++ thread::sleep(Duration::from_secs(3)); // window + electron warmup++ let mut corners = vec![+ Corner { title: "Win32 GDI", session: "crimson", hwnd: HWND::default(), pid: 0, field_idx: None, submit_idx: None },+ Corner { title: "WinForms", session: "amber", hwnd: HWND::default(), pid: 0, field_idx: None, submit_idx: None },+ Corner { title: "WPF", session: "aqua", hwnd: HWND::default(), pid: 0, field_idx: None, submit_idx: None },+ Corner { title: "Electron", session: "mint_lime", hwnd: HWND::default(), pid: 0, field_idx: None, submit_idx: None },+ ];+ for c in corners.iter_mut() {+ if let Some((h, pid)) = find_window_by_title(c.title) { c.hwnd = h; c.pid = pid; }+ else { eprintln!("[orch] (skip) no window for '{}'", c.title); }+ }+ corners.retain(|c| !c.hwnd.0.is_null());+ let master_hwnd = find_window_by_title("Master (Win32").map(|(h, _)| h);++ // Layout: each window half the work area; corners tile quadrants, master centered on top.+ let wa = work_area();+ let ww = (wa.right - wa.left) / 2;+ let wh = (wa.bottom - wa.top) / 2;+ let positions = [(wa.left, wa.top), (wa.left + ww, wa.top), (wa.left, wa.top + wh), (wa.left + ww, wa.top + wh)];+ for (i, c) in corners.iter().enumerate() { let (x, y) = positions[i % 4]; place(c.hwnd, x, y, ww, wh, false); }+ if let Some(mh) = master_hwnd {+ // Master is smaller than a quadrant so it overlaps the central seam of+ // all four corners without hiding their forms (which sit at the+ // quadrant centers, outside this box). Centered on the work area.+ let (mw, mh2) = (ww / 2, wh / 2);+ let (mx, my) = (wa.left + ww - mw / 2, wa.top + wh - mh2 / 2);+ place(mh, mx, my, mw, mh2, true);+ unsafe { let _ = SetForegroundWindow(mh); }+ }+ thread::sleep(Duration::from_millis(800)); // let resized layouts settle++ // Discover UIA elements per corner (field + SUBMIT). GDI has none -> pixel.+ for c in corners.iter_mut() {+ let tree = run_call_out(&cua, "get_window_state",+ &format!(r#"{{"pid":{},"window_id":{},"capture_mode":"ax","session":"{}"}}"#, c.pid, c.hwnd.0 as isize, c.session));+ c.field_idx = find_idx(&tree, |line| line.contains("] Edit"));+ c.submit_idx = find_idx(&tree, |line| line.contains("] Button") && line.to_uppercase().contains("ADD RECORD"));+ eprintln!("[orch] {:<9} session={:<9} field={:?} submit={:?} {}",+ c.title, c.session, c.field_idx, c.submit_idx,+ if c.field_idx.is_some() { "(a11y: set_value+invoke)" } else { "(no a11y: pixel)" });+ // Pre-arm a coloured cursor for this session.+ let _ = run_call(&cua, "set_agent_cursor_enabled", &format!(r#"{{"enabled":true,"session":"{}"}}"#, c.session));+ }++ // One driver thread per corner.+ let mut senders: Vec<Sender<Action>> = Vec::new();+ let mut handles = Vec::new();+ for c in &corners {+ let (tx, rx) = channel::<Action>();+ senders.push(tx);+ let cua = cua.clone();+ let pid = c.pid;+ let hwnd_addr = c.hwnd.0 as isize;+ let session = c.session.to_string();+ let title = c.title.to_string();+ handles.push(thread::spawn(move || {+ let hwnd = HWND(hwnd_addr as *mut _);+ let ax = format!(r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"ax","session":"{session}"}}"#);+ let mut seen_records: i64 = -1;+ for act in rx {+ // Re-discover element indices on a FRESH snapshot each action β+ // a11y vs pixel is decided per action (Chromium's tree appears+ // late and re-numbers as rows are added; GDI never has a tree).+ let tree = run_call_out(&cua, "get_window_state", &ax);+ let field = find_idx(&tree, |l| l.contains("] Edit"));+ let submit = find_idx(&tree, |l| l.contains("] Button") && l.to_uppercase().contains("ADD RECORD"));+ match act {+ Action::Type { text } => {+ let ok = match field {+ // a11y: plain type_text with element_index β cua-driver+ // auto-routes to UIA ValuePattern.SetValue on its own.+ Some(idx) => run_call(&cua, "type_text", &format!(+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"element_index":{idx},"text":"{}","session":"{session}"}}"#, json_escape(&text))),+ // no a11y (GDI): focus the field by pixel, then WM_CHAR.+ None => {+ if let Some((x, y)) = client_rel_to_local_px(hwnd, FIELD_FRAC.0, FIELD_FRAC.1) {+ let _ = run_call(&cua, "click", &format!(+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"x":{x},"y":{y},"session":"{session}"}}"#));+ }+ run_call(&cua, "type_text", &format!(+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"text":"{}","session":"{session}"}}"#, json_escape(&text)))+ }+ };+ eprintln!("[drive {session}] type {text:?} -> {}", if ok { "ok" } else { "FAIL" });+ }+ Action::Click { rx, ry } => {+ let ok = match submit {+ Some(idx) => run_call(&cua, "click", &format!(+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"element_index":{idx},"session":"{session}"}}"#)),+ None => {+ if let Some((x, y)) = client_rel_to_local_px(hwnd, rx, ry) {+ run_call(&cua, "click", &format!(+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"x":{x},"y":{y},"session":"{session}"}}"#))+ } else { false }+ }+ };+ eprintln!("[drive {session}] click Add Record -> {}", if ok { "ok" } else { "FAIL" });++ // Verify a record was actually COMMITTED: the "Records: N"+ // counter must increment. (Checking for the typed text+ // alone is fooled by it sitting in the field.) Only the+ // a11y corners expose the counter; GDI's is drawn pixels,+ // so it's verified visually (its grid grows on screen).+ thread::sleep(Duration::from_millis(300));+ let t2 = run_call_out(&cua, "get_window_state", &ax);+ if let Some(n) = records_count(&t2) {+ let landed = n as i64 > seen_records;+ seen_records = n as i64;+ eprintln!("[verify {session}] {title} Records={n} -> {}",+ if landed { "COMMITTED β" } else { "FAIL (no new record)" });+ }+ }+ Action::DrawMaze => {+ // Pure coordinate drawing β NO element targeting, NO app+ // cooperation. Each maze segment is one straight+ // press-drag-release into the common REGION; cua-driver+ // PostMessages the drag where it can and pen-injects it+ // where the canvas (Chromium/WPF) drops posted mouse.+ let map = |mx: f64, my: f64| {+ let cfx = REGION[0] + mx * (REGION[2] - REGION[0]);+ let cfy = REGION[1] + my * (REGION[3] - REGION[1]);+ client_rel_to_local_px(hwnd, cfx, cfy)+ };+ let mut drawn = 0;+ for (x0, y0, x1, y1) in MAZE.iter() {+ if let (Some((fx, fy)), Some((tx, ty))) = (map(*x0, *y0), map(*x1, *y1)) {+ if run_call(&cua, "drag", &format!(+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"from_x":{fx},"from_y":{fy},"to_x":{tx},"to_y":{ty},"dispatch":"background","session":"{session}"}}"#)) {+ drawn += 1;+ }+ }+ thread::sleep(Duration::from_millis(120));+ }+ eprintln!("[draw {session}] maze: {drawn}/{} segments", MAZE.len());++ // Read back by SCREENSHOT (WGC captures the occluded+ // window) and diff the rendered ink against the+ // rasterized reference maze β normalized into the same+ // REGION grid, pass only under threshold.+ thread::sleep(Duration::from_millis(400));+ let shot = std::env::temp_dir().join(format!("maze_{session}.png"));+ let _ = std::fs::remove_file(&shot);+ run_call_shot(&cua, &format!(+ r#"{{"pid":{pid},"window_id":{hwnd_addr},"capture_mode":"vision","max_image_dimension":4096,"session":"{session}"}}"#), &shot);+ match score_maze(hwnd, &shot) {+ Some(score) => eprintln!("[verify {session}] {title} maze score={score:.2} -> {}",+ if score >= MAZE_PASS { "DREW MAZE β" } else { "FAIL (lines don't match)" }),+ None => eprintln!("[verify {session}] {title} maze -> FAIL (no screenshot/region)"),+ }+ }+ }+ }+ }));+ }++ eprintln!("[orch] ready β submit a record in the center MASTER; watch {} coloured cursors \+ drive the corner terminals in the background.", corners.len());++ if std::env::args().any(|a| a == "--auto") {+ let s2 = senders.clone();+ thread::spawn(move || {+ for i in 1..=3 {+ thread::sleep(Duration::from_secs(3));+ eprintln!("[orch] AUTO {i}: TYPE then CLICK");+ for tx in &s2 { let _ = tx.send(Action::Type { text: format!("SUBJECT-{i:03}") }); }+ thread::sleep(Duration::from_millis(1800));+ // SAVE/"Add Record" button center (matches the GDI/master layout)+ // for the pixel-driven GDI corner; a11y corners ignore rx/ry.+ for tx in &s2 { let _ = tx.send(Action::Click { rx: 0.22, ry: 0.495 }); }+ }+ // Finally: draw the same maze into every corner's drawing pad by+ // pure coordinate drags, then screenshot-diff each against the+ // reference. Proves canvas actuation with no a11y, no app help.+ thread::sleep(Duration::from_secs(2));+ eprintln!("[orch] AUTO 4: DRAW MAZE (line tool) + screenshot diff");+ for tx in &s2 { let _ = tx.send(Action::DrawMaze); }+ });+ }++ let reader = BufReader::new(master_out);+ for line in reader.lines().map_while(Result::ok) {+ let parts: Vec<&str> = line.trim().split('\t').collect();+ let action = match parts.as_slice() {+ ["CLICK", rx, ry] => rx.parse::<f64>().ok().zip(ry.parse::<f64>().ok()).map(|(rx, ry)| Action::Click { rx, ry }),+ ["TYPE", text] => Some(Action::Type { text: (*text).to_string() }),+ _ => None,+ };+ if let Some(a) = action {+ eprintln!("[orch] user action: {a:?} -> driving {} corners", senders.len());+ for tx in &senders { let _ = tx.send(a.clone()); }+ }+ }++ drop(senders);+ for h in handles { let _ = h.join(); }+ let _ = master.kill();+ for mut k in kids { let _ = k.kill(); }+ let _ = daemon.kill();+}++#[derive(Clone, Debug)]+enum Action { Click { rx: f64, ry: f64 }, Type { text: String }, DrawMaze }++fn json_escape(s: &str) -> String {+ let mut o = String::with_capacity(s.len());+ for ch in s.chars() {+ match ch { '"' => o.push_str("\\\""), '\\' => o.push_str("\\\\"), '\n' => o.push_str("\\n"),+ '\r' => {}, '\t' => o.push_str("\\t"), c => o.push(c) }+ }+ o+}++/// `cua-driver call <tool> <json>` (proxies to the daemon). True on success.+fn run_call(cua: &PathBuf, tool: &str, json: &str) -> bool {+ Command::new(cua).arg("call").arg(tool).arg(json)+ .stdout(Stdio::null()).stderr(Stdio::null())+ .status().map(|s| s.success()).unwrap_or(false)+}+/// Same, capturing stdout (for get_window_state readback).+fn run_call_out(cua: &PathBuf, tool: &str, json: &str) -> String {+ Command::new(cua).arg("call").arg(tool).arg(json)+ .stderr(Stdio::null()).output()+ .map(|o| String::from_utf8_lossy(&o.stdout).into_owned()).unwrap_or_default()+}+/// get_window_state writing the screenshot PNG to `out` (WGC; captures even a+/// fully occluded window). Used to read back what cua-driver drew.+fn run_call_shot(cua: &PathBuf, json: &str, out: &std::path::Path) -> bool {+ Command::new(cua).arg("call").arg("get_window_state").arg(json)+ .arg("--screenshot-out-file").arg(out)+ .stdout(Stdio::null()).stderr(Stdio::null())+ .status().map(|s| s.success()).unwrap_or(false)+}++/// DWM extended-frame size of `hwnd` β the pixel size of the WGC bitmap before+/// get_window_state downscales it to fit `max_image_dimension`.+fn window_frame_size(hwnd: HWND) -> Option<(i32, i32)> {+ unsafe {+ let mut d = RECT::default();+ if DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &mut d as *mut _ as *mut core::ffi::c_void, std::mem::size_of::<RECT>() as u32).is_err() {+ return None;+ }+ Some((d.right - d.left, d.bottom - d.top))+ }+}++/// Shape-only F1 (0..1) of the maze cua-driver drew vs the reference maze.+///+/// Reads the ink from the window screenshot (cropped to the drawing REGION),+/// then normalizes BOTH the drawn ink and the reference into a unit grid by+/// their own bounding boxes β so the score reflects the *shape* of the lines,+/// not where on the canvas they landed or at what scale (the user only wants+/// the shape penalized). The screenshot is downscaled to fit+/// `max_image_dimension`, so REGION (full-res bitmap px) is scaled by the+/// PNG/frame ratio before cropping.+fn score_maze(hwnd: HWND, png: &std::path::Path) -> Option<f64> {+ let img = image::open(png).ok()?.to_luma8();+ let (iw, ih) = (img.width() as i32, img.height() as i32);+ let (fw, fh) = window_frame_size(hwnd)?;+ if fw <= 0 || fh <= 0 { return None; }+ let (fx, fy) = (iw as f64 / fw as f64, ih as f64 / fh as f64);+ let (ax, ay) = client_rel_to_local_px(hwnd, REGION[0], REGION[1])?;+ let (bx, by) = client_rel_to_local_px(hwnd, REGION[2], REGION[3])?;+ let cx0 = (((ax.min(bx)) as f64 * fx).floor() as i32).max(0);+ let cy0 = (((ay.min(by)) as f64 * fy).floor() as i32).max(0);+ let cx1 = (((ax.max(bx)) as f64 * fx).ceil() as i32).min(iw - 1);+ let cy1 = (((ay.max(by)) as f64 * fy).ceil() as i32).min(ih - 1);+ if cx1 - cx0 < 12 || cy1 - cy0 < 12 { return None; }++ // Drawn ink + its bounding box.+ let mut ink: Vec<(i32, i32)> = Vec::new();+ let (mut nx, mut ny, mut xx, mut xy) = (i32::MAX, i32::MAX, i32::MIN, i32::MIN);+ for y in cy0..=cy1 {+ for x in cx0..=cx1 {+ if img.get_pixel(x as u32, y as u32).0[0] < 150 {+ ink.push((x, y));+ if x < nx { nx = x } if x > xx { xx = x }+ if y < ny { ny = y } if y > xy { xy = y }+ }+ }+ }+ const N: usize = 96;+ if ink.len() < 20 || xx <= nx || xy <= ny { return None; }+ let (bw, bh) = ((xx - nx) as f64, (xy - ny) as f64);+ let mut a = vec![false; N * N];+ for (x, y) in ink {+ let gx = (((x - nx) as f64 / bw) * (N as f64 - 1.0)).round() as usize;+ let gy = (((y - ny) as f64 / bh) * (N as f64 - 1.0)).round() as usize;+ a[gy.min(N - 1) * N + gx.min(N - 1)] = true;+ }++ // Reference maze, normalized to ITS bounding box the same way.+ let (mut rnx, mut rny, mut rxx, mut rxy) = (f64::MAX, f64::MAX, f64::MIN, f64::MIN);+ for (x0, y0, x1, y1) in MAZE.iter() {+ for (px, py) in [(*x0, *y0), (*x1, *y1)] {+ if px < rnx { rnx = px } if px > rxx { rxx = px }+ if py < rny { rny = py } if py > rxy { rxy = py }+ }+ }+ let (rbw, rbh) = ((rxx - rnx).max(1e-6), (rxy - rny).max(1e-6));+ let mut b = vec![false; N * N];+ for (x0, y0, x1, y1) in MAZE.iter() {+ raster_line(&mut b, N, (x0 - rnx) / rbw, (y0 - rny) / rbh, (x1 - rnx) / rbw, (y1 - rny) / rbh);+ }+ Some(mask_f1(&a, &b, N, 2))+}++/// Bresenham rasterize a normalized [0,1] segment into the NΓN reference mask.+fn raster_line(mask: &mut [bool], n: usize, x0: f64, y0: f64, x1: f64, y1: f64) {+ let s = n as f64 - 1.0;+ let (px1, py1) = ((x1 * s) as i32, (y1 * s) as i32);+ let (mut x, mut y) = ((x0 * s) as i32, (y0 * s) as i32);+ let (dx, dy) = ((px1 - x).abs(), -(py1 - y).abs());+ let (sx, sy) = (if x < px1 { 1 } else { -1 }, if y < py1 { 1 } else { -1 });+ let mut err = dx + dy;+ loop {+ if x >= 0 && y >= 0 && (x as usize) < n && (y as usize) < n { mask[y as usize * n + x as usize] = true; }+ if x == px1 && y == py1 { break; }+ let e2 = 2 * err;+ if e2 >= dy { err += dy; x += sx; }+ if e2 <= dx { err += dx; y += sy; }+ }+}++/// Chebyshev dilation by radius `r`.+fn dilate(m: &[bool], n: usize, r: i32) -> Vec<bool> {+ let mut o = vec![false; n * n];+ for y in 0..n as i32 {+ for x in 0..n as i32 {+ if !m[y as usize * n + x as usize] { continue; }+ for dy in -r..=r {+ for dx in -r..=r {+ let (nx, ny) = (x + dx, y + dy);+ if nx >= 0 && ny >= 0 && (nx as usize) < n && (ny as usize) < n { o[ny as usize * n + nx as usize] = true; }+ }+ }+ }+ }+ o+}++/// F1 of two ink masks, each dilated by `r` so near-misses count as matches.+fn mask_f1(a: &[bool], b: &[bool], n: usize, r: i32) -> f64 {+ let (da, db) = (dilate(a, n, r), dilate(b, n, r));+ let (mut ca, mut cb, mut ma, mut mb) = (0usize, 0usize, 0usize, 0usize);+ for i in 0..n * n {+ if a[i] { ca += 1; if db[i] { ma += 1; } }+ if b[i] { cb += 1; if da[i] { mb += 1; } }+ }+ if ca == 0 || cb == 0 { return 0.0; }+ let precision = ma as f64 / ca as f64; // drawn ink that is near the reference+ let recall = mb as f64 / cb as f64; // reference covered by drawn ink+ if precision + recall == 0.0 { 0.0 } else { 2.0 * precision * recall / (precision + recall) }+}++/// First `[N]` element index on a tree line matching `pred`.+fn find_idx(tree: &str, pred: impl Fn(&str) -> bool) -> Option<u64> {+ // The tree is one JSON line with literal "\n" separators between rows.+ for line in tree.split("\\n") {+ if !pred(line) { continue; }+ let st = line.find('[')? + 1;+ let en = line[st..].find(']')? + st;+ if let Ok(n) = line[st..en].trim().parse() { return Some(n); }+ }+ None+}++/// Parse the "Records: N" counter from a window-state tree (status bar).+fn records_count(tree: &str) -> Option<u32> {+ let pos = tree.find("Records:")?;+ let rest = &tree[pos + "Records:".len()..];+ let digits: String = rest.trim_start().chars().take_while(|c| c.is_ascii_digit()).collect();+ digits.parse().ok()+}++fn work_area() -> RECT {+ use windows::Win32::UI::WindowsAndMessaging::{SystemParametersInfoW, SPI_GETWORKAREA, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS};+ let mut wa = RECT::default();+ unsafe { let _ = SystemParametersInfoW(SPI_GETWORKAREA, 0, Some(&mut wa as *mut _ as *mut core::ffi::c_void), SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0)); }+ if wa.right <= wa.left || wa.bottom <= wa.top {+ use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN};+ unsafe { wa = RECT { left: 0, top: 0, right: GetSystemMetrics(SM_CXSCREEN), bottom: GetSystemMetrics(SM_CYSCREEN) }; }+ }+ wa+}++fn place(hwnd: HWND, x: i32, y: i32, w: i32, h: i32, activate: bool) {+ if hwnd.0.is_null() { return; }+ let flags = if activate { SWP_SHOWWINDOW } else { SWP_NOACTIVATE | SWP_SHOWWINDOW | SWP_NOZORDER };+ unsafe { let _ = SetWindowPos(hwnd, HWND_TOP, x, y, w, h, flags); }+}++/// Client-relative (0..1) -> the click tool's window-local screenshot-pixel+/// space (ClientToScreen minus the DWM extended-frame top-left + 1px inset).+fn client_rel_to_local_px(hwnd: HWND, rx: f64, ry: f64) -> Option<(i32, i32)> {+ unsafe {+ let mut cr = RECT::default();+ GetClientRect(hwnd, &mut cr).ok()?;+ let mut pt = POINT { x: (rx * (cr.right - cr.left) as f64) as i32, y: (ry * (cr.bottom - cr.top) as f64) as i32 };+ let _ = ClientToScreen(hwnd, &mut pt);+ let mut dwm = RECT::default();+ if DwmGetWindowAttribute(hwnd, DWMWA_EXTENDED_FRAME_BOUNDS, &mut dwm as *mut _ as *mut core::ffi::c_void, std::mem::size_of::<RECT>() as u32).is_err() {+ return Some((pt.x, pt.y));+ }+ Some((pt.x - dwm.left - 1, pt.y - dwm.top - 1))+ }+}++// ββ window discovery by title substring (case-insensitive) ββββββββββββββββββββ+struct Finder { needle: String, hwnd: HWND, pid: u32 }+unsafe extern "system" fn enum_cb(hwnd: HWND, lparam: LPARAM) -> BOOL {+ let f = &mut *(lparam.0 as *mut Finder);+ if !IsWindowVisible(hwnd).as_bool() { return TRUE; }+ let mut buf = [0u16; 256];+ let n = GetWindowTextW(hwnd, &mut buf);+ if n > 0 {+ let title = String::from_utf16_lossy(&buf[..n as usize]);+ if title.to_lowercase().contains(&f.needle.to_lowercase()) {+ let mut pid = 0u32;+ GetWindowThreadProcessId(hwnd, Some(&mut pid));+ f.hwnd = hwnd; f.pid = pid;+ return BOOL(0);+ }+ }+ let _ = PWSTR::null();+ TRUE+}+fn find_window_by_title(needle: &str) -> Option<(HWND, u32)> {+ let deadline = Instant::now() + Duration::from_secs(8);+ loop {+ let mut f = Finder { needle: needle.to_string(), hwnd: HWND::default(), pid: 0 };+ unsafe { let _ = EnumWindows(Some(enum_cb), LPARAM(&mut f as *mut _ as isize)); }+ if !f.hwnd.0.is_null() { return Some((f.hwnd, f.pid)); }+ if Instant::now() > deadline { return None; }+ thread::sleep(Duration::from_millis(300));+ }+}diff --git a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver/Cargo.tomlindex affa006490..57156b1f29 100644--- a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml+++ b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml@@ -69,4 +69,15 @@ tempfile = "3"[target.'cfg(target_os = "windows")'.dev-dependencies]platform-windows = { path = "../platform-windows" }-windows = { version = "0.61", features = ["Win32_UI_WindowsAndMessaging", "Win32_Foundation"] }+windows = { version = "0.61", features = [+ "Win32_UI_WindowsAndMessaging",+ "Win32_Foundation",+ # Job Object: assign every spawned test child to a KILL_ON_JOB_CLOSE job so+ # the OS terminates the whole tree when the test process dies for ANY reason+ # (including SIGKILL / Ctrl-C) β guarantees no orphaned windows or held ports.+ "Win32_System_JobObjects",+ # CreateJobObjectW's signature references SECURITY_ATTRIBUTES;+ # JOBOBJECT_EXTENDED_LIMIT_INFORMATION embeds IO_COUNTERS (Threading).+ "Win32_Security",+ "Win32_System_Threading",+] }diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/e2e_windows_bg_input_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/e2e_windows_bg_input_test.rsnew file mode 100644index 0000000000..c0a1a66555--- /dev/null+++ b/libs/cua-driver/rust/crates/cua-driver/tests/e2e_windows_bg_input_test.rs@@ -0,0 +1,558 @@+//! End-to-end Windows test for the **unified background input interface**:+//! a caller targets an app and plays click/key actions WITHOUT knowing the+//! app's internals (Electron/Chromium, Tauri/WebView2, classic Win32) and+//! WITHOUT the target window ever being raised to the foreground.+//!+//! Verified two ways per action:+//! 1. The tool succeeds in the DEFAULT dispatch mode β no+//! `background_unavailable` error, no `dispatch:"foreground"` needed.+//! 2. The `focus-monitor-win` sentinel records ZERO foreground losses across+//! the action == the target window was not z-raised over the user's+//! window (same oracle as `harness_bg_modality_test`).+//!+//! ## Process hygiene (no orphaned windows / held ports)+//! Every child (target app, sentinel, driver) is spawned into a Windows **Job+//! Object** created with `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`. The test process+//! holds the only job handle, so when it exits for ANY reason β normal end,+//! panic, or being force-killed (Ctrl-C) β the OS terminates the entire child+//! tree. A per-fixture `ChildBag` also kills+waits promptly between tests so+//! windows don't pile up during a run, and every early-return path reaps.+//!+//! Targets (trycua test apps, auto-downloaded to %TEMP% if not provided):+//! - Electron (Chromium content) β `trycua/desktop-test-app-electron`.+//! - Tauri (WebView2 content) β `trycua/desktop-test-app`.+//! - Win32 baseline β `notepad.exe`.+//! Override with `CUA_ELECTRON_EXE` / `CUA_TAURI_EXE`, or drop the exe in+//! `test-apps/`. Auto-download uses `curl.exe`.+//!+//! Every JSON-RPC call is bounded by a hard timeout (a hung driver becomes a+//! fast, localized failure β never an indefinite wall-clock hang).+//!+//! All tests are `#[ignore]` (GUI, real desktop session). Run explicitly,+//! serially (apps bind a fixed HTTP port, so never in parallel):+//! cargo test -p cua-driver --test e2e_windows_bg_input_test -- --ignored --nocapture --test-threads=1++#![cfg(target_os = "windows")]++use core::ffi::c_void;+use std::collections::HashSet;+use std::fs;+use std::io::{BufRead, BufReader, Write};+use std::os::windows::io::AsRawHandle;+use std::path::{Path, PathBuf};+use std::process::{Child, ChildStdin, Command, Stdio};+use std::sync::mpsc::{channel, Receiver};+use std::sync::OnceLock;+use std::time::{Duration, Instant};++use windows::Win32::Foundation::{CloseHandle, HANDLE};+use windows::Win32::System::JobObjects::{+ AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject,+ JobObjectExtendedLimitInformation, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,+ JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,+};+use windows::Win32::System::Threading::{OpenProcess, PROCESS_SET_QUOTA, PROCESS_TERMINATE};++const CALL_TIMEOUT: Duration = Duration::from_secs(25);++const ELECTRON_URL: &str = "https://github.com/trycua/desktop-test-app-electron/releases/download/v0.1.0/desktop-test-app-electron.0.1.0.exe";+const TAURI_URL: &str = "https://github.com/trycua/desktop-test-app/releases/download/v0.2.2/desktop-test-app-windows-x86_64.exe";++// ββ Job Object: kill the whole child tree when this test process dies βββββββββ++/// Process-global job handle (stored as usize so it's `Send`/`Sync` in the+/// OnceLock). Created lazily with KILL_ON_JOB_CLOSE.+static JOB: OnceLock<usize> = OnceLock::new();++fn job() -> HANDLE {+ let raw = *JOB.get_or_init(|| unsafe {+ let h = CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW");+ let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();+ info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;+ let _ = SetInformationJobObject(+ h,+ JobObjectExtendedLimitInformation,+ &info as *const _ as *const c_void,+ std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,+ );+ h.0 as usize+ });+ HANDLE(raw as *mut c_void)+}++/// Spawn a command and immediately assign it to the kill-on-close job, so it+/// can never outlive this test process.+fn spawn_in_job(cmd: &mut Command) -> std::io::Result<Child> {+ let child = cmd.spawn()?;+ unsafe {+ let h = HANDLE(child.as_raw_handle() as *mut c_void);+ let _ = AssignProcessToJobObject(job(), h);+ }+ Ok(child)+}++/// Assign an already-running pid (e.g. the broker-spawned window process of a+/// packaged app, which is NOT our direct child) to the kill-on-close job, so+/// it too dies when this test process exits. Best-effort.+fn assign_pid_to_job(pid: u32) {+ unsafe {+ if let Ok(h) = OpenProcess(PROCESS_SET_QUOTA | PROCESS_TERMINATE, false, pid) {+ if !h.is_invalid() {+ let _ = AssignProcessToJobObject(job(), h);+ let _ = CloseHandle(h);+ }+ }+ }+}++/// Owns every spawned child AND any externally-launched pids (broker apps);+/// kills them all on drop β prompt cleanup between tests and on early+/// return / panic unwind. The Job Object is the backstop for hard kills.+struct ChildBag { children: Vec<Child>, pids: Vec<u32> }+impl ChildBag {+ fn new() -> Self { ChildBag { children: Vec::new(), pids: Vec::new() } }+ fn push(&mut self, c: Child) { self.children.push(c); }+ /// Track an external pid (and its whole tree) for teardown via taskkill.+ fn track_pid(&mut self, pid: u32) { self.pids.push(pid); }+}+impl Drop for ChildBag {+ fn drop(&mut self) {+ // Tree-kill externally-discovered window processes first (packaged /+ // broker-launched apps whose window pid isn't our spawned child).+ for pid in &self.pids {+ let _ = Command::new("taskkill")+ .args(["/F", "/T", "/PID", &pid.to_string()])+ .stdout(Stdio::null()).stderr(Stdio::null())+ .status();+ }+ for c in &mut self.children {+ let _ = c.kill();+ let _ = c.wait();+ }+ std::thread::sleep(Duration::from_millis(250));+ }+}++/// Best-effort kill of any prior instance of `exe` by basename, so a leftover+/// from an earlier (e.g. force-killed) run can't hold the app's fixed HTTP port.+fn kill_prior_by_name(exe: &Path) {+ if let Some(name) = exe.file_name().and_then(|n| n.to_str()) {+ let _ = Command::new("taskkill")+ .args(["/F", "/T", "/IM", name])+ .stdout(Stdio::null()).stderr(Stdio::null())+ .status();+ }+}++// ββ paths / downloads βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++fn workspace_root() -> PathBuf {+ let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR");+ PathBuf::from(manifest).parent().unwrap().parent().unwrap().to_owned()+}+fn driver_binary() -> PathBuf { workspace_root().join("target/debug/cua-driver.exe") }+fn focus_monitor_binary() -> PathBuf { workspace_root().join("target/debug/focus-monitor-win.exe") }++fn curl_download(url: &str, dst: &Path) -> bool {+ eprintln!("[e2e] downloading {url}\n -> {dst:?}");+ let ok = Command::new("curl.exe")+ .args(["-L", "--fail", "--silent", "--show-error", "-o"])+ .arg(dst).arg(url).status().map(|s| s.success()).unwrap_or(false);+ ok && fs::metadata(dst).map(|m| m.len() > 4096).unwrap_or(false)+}+fn resolve_or_download(env_var: &str, prefix: &str, url: &str, tmp_name: &str) -> Option<PathBuf> {+ if let Ok(p) = std::env::var(env_var) {+ let pb = PathBuf::from(p);+ if pb.exists() { return Some(pb); }+ }+ if let Ok(entries) = fs::read_dir(workspace_root().join("test-apps")) {+ for e in entries.flatten() {+ let name = e.file_name().to_string_lossy().to_lowercase();+ if name.starts_with(prefix) && name.ends_with(".exe") && !name.contains("setup") {+ return Some(e.path());+ }+ }+ }+ let dst = std::env::temp_dir().join(tmp_name);+ if dst.exists() && fs::metadata(&dst).map(|m| m.len() > 4096).unwrap_or(false) {+ return Some(dst);+ }+ if curl_download(url, &dst) { Some(dst) } else { None }+}+fn electron_exe() -> Option<PathBuf> {+ resolve_or_download("CUA_ELECTRON_EXE", "desktop-test-app-electron", ELECTRON_URL,+ "desktop-test-app-electron.0.1.0.exe")+}+fn tauri_exe() -> Option<PathBuf> {+ resolve_or_download("CUA_TAURI_EXE", "desktop-test-app-windows", TAURI_URL,+ "desktop-test-app-windows-x86_64.exe")+}++fn loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_losses.txt") }+fn key_loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_losses.txt") }+fn focus_pid_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_pid.txt") }+fn focus_hwnd_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_hwnd.txt") }+fn read_count(p: &Path) -> u32 {+ fs::read_to_string(p).ok().and_then(|s| s.trim().parse::<u32>().ok()).unwrap_or(0)+}++// ββ JSON-RPC over the driver's stdio, with per-call timeout βββββββββββββββββββ++fn send(stdin: &mut ChildStdin, req: serde_json::Value) {+ let _ = writeln!(stdin, "{}", serde_json::to_string(&req).unwrap());+ let _ = stdin.flush();+}+fn call(stdin: &mut ChildStdin, rx: &Receiver<String>,+ id: u32, name: &str, args: serde_json::Value) -> serde_json::Value {+ send(stdin, serde_json::json!({+ "jsonrpc":"2.0","id":id,"method":"tools/call","params":{"name":name,"arguments":args}+ }));+ match rx.recv_timeout(CALL_TIMEOUT) {+ Ok(line) => serde_json::from_str(&line)+ .unwrap_or_else(|_| serde_json::json!({"error": format!("bad json: {line}")})),+ Err(_) => serde_json::json!({"error": format!("TIMEOUT (>{}s) on {name}", CALL_TIMEOUT.as_secs())}),+ }+}+fn init(stdin: &mut ChildStdin, rx: &Receiver<String>) {+ send(stdin, serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}));+ let _ = rx.recv_timeout(CALL_TIMEOUT);+}+fn result_text(s: &serde_json::Value) -> String {+ s["result"]["content"][0]["text"].as_str().unwrap_or("").to_string()+}+fn is_error(s: &serde_json::Value) -> bool {+ s["result"]["isError"].as_bool().unwrap_or(false) || s.get("error").is_some()+}+fn find_idx_containing(s: &serde_json::Value, needle: &str) -> Option<u64> {+ for line in result_text(s).lines() {+ if !line.to_lowercase().contains(&needle.to_lowercase()) { continue; }+ let st = line.find('[')? + 1;+ let en = line[st..].find(']')? + st;+ if let Ok(n) = line[st..en].trim().parse() { return Some(n); }+ }+ None+}++fn window_ids(stdin: &mut ChildStdin, rx: &Receiver<String>) -> HashSet<u64> {+ let r = call(stdin, rx, 99, "list_windows", serde_json::json!({}));+ r["result"]["structuredContent"]["windows"].as_array()+ .map(|a| a.iter().filter_map(|w| w["window_id"].as_u64()).collect())+ .unwrap_or_default()+}++// ββ fixture βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++struct E2eFixture {+ _bag: ChildBag, // kills target+sentinel+driver on drop+ stdin: ChildStdin,+ rx: Receiver<String>,+ pid: u32,+ wid: u64,+}++/// Launch order: driver (no window) β snapshot windows β app β discover the+/// app's NEW window (works for multi-process apps like Electron whose window+/// belongs to a child pid) β sentinel (grabs foreground, pushing the app to+/// the background) β reset counters.+fn setup(target_exe: &Path, _title_hint: &str) -> Option<E2eFixture> {+ let driver_bin = driver_binary();+ let fm_bin = focus_monitor_binary();+ if !driver_bin.exists() { eprintln!("[e2e] cua-driver.exe not built β skipping"); return None; }+ if !fm_bin.exists() { eprintln!("[e2e] focus-monitor-win.exe not built β skipping"); return None; }+ if target_exe.is_absolute() && !target_exe.exists() {+ eprintln!("[e2e] target {target_exe:?} missing β skipping"); return None;+ }++ // Defensive: clear any leftover instance holding the app's fixed port.+ kill_prior_by_name(target_exe);+ let _ = fs::write(loss_file(), "0");+ let _ = fs::write(key_loss_file(), "0");+ let _ = fs::remove_file(focus_pid_file());+ let _ = fs::remove_file(focus_hwnd_file());++ let mut bag = ChildBag::new();++ // 1. Driver (daemon, no visible window).+ let mut driver = spawn_in_job(+ Command::new(&driver_bin).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())+ ).inspect_err(|e| eprintln!("[e2e] driver spawn failed: {e}")).ok()?;+ let mut stdin = driver.stdin.take().unwrap();+ let stdout = driver.stdout.take().unwrap();+ bag.push(driver);+ let (tx, rx) = channel::<String>();+ std::thread::spawn(move || {+ let mut reader = BufReader::new(stdout);+ let mut line = String::new();+ loop {+ line.clear();+ match reader.read_line(&mut line) {+ Ok(0) | Err(_) => break,+ Ok(_) => { if tx.send(line.trim().to_string()).is_err() { break; } }+ }+ }+ });+ init(&mut stdin, &rx);++ // 2. Snapshot existing windows, then launch the target app.+ let before = window_ids(&mut stdin, &rx);+ let app = match spawn_in_job(+ Command::new(target_exe).stdout(Stdio::null()).stderr(Stdio::null())+ ) {+ Ok(c) => c,+ Err(e) => { eprintln!("[e2e] target spawn failed: {e}"); return None; } // bag drops β kills driver+ };+ bag.push(app);++ // 3. Discover the app's NEW window (its pid may be a child of the launched+ // process β Electron/WebView2 are multi-process).+ let deadline = Instant::now() + Duration::from_secs(15);+ let mut found: Option<(u32, u64)> = None;+ while Instant::now() < deadline {+ let r = call(&mut stdin, &rx, 11, "list_windows", serde_json::json!({}));+ if let Some(arr) = r["result"]["structuredContent"]["windows"].as_array() {+ for w in arr {+ let Some(wid) = w["window_id"].as_u64() else { continue };+ let pid = w["pid"].as_u64().unwrap_or(0) as u32;+ let title = w["title"].as_str().unwrap_or("");+ if !before.contains(&wid) && !title.is_empty() && pid != 0 {+ found = Some((pid, wid));+ break;+ }+ }+ }+ if found.is_some() { break; }+ std::thread::sleep(Duration::from_millis(500));+ }+ let (pid, wid) = match found {+ Some(p) => p,+ None => { eprintln!("[e2e] app window never appeared β skipping"); return None; } // bag drops+ };+ // The window's process is often broker-spawned (packaged apps, Electron),+ // i.e. NOT our direct child. Job-assign it (hard-kill safety) and track it+ // for prompt tree-kill on teardown so it can't orphan.+ assign_pid_to_job(pid);+ bag.track_pid(pid);++ // 4. Sentinel grabs foreground; app drops to z+1 (the background target).+ let fm = match spawn_in_job(+ Command::new(&fm_bin).stdout(Stdio::null()).stderr(Stdio::null())+ ) {+ Ok(c) => c,+ Err(e) => { eprintln!("[e2e] sentinel spawn failed: {e}"); return None; }+ };+ bag.push(fm);+ let sdeadline = Instant::now() + Duration::from_secs(10);+ loop {+ let ok = read_count(&focus_pid_file()) != 0+ && fs::read_to_string(focus_hwnd_file()).ok()+ .and_then(|s| s.trim().parse::<u64>().ok()).unwrap_or(0) != 0;+ if ok { break; }+ if Instant::now() > sdeadline { eprintln!("[e2e] sentinel never published β skipping"); return None; }+ std::thread::sleep(Duration::from_millis(100));+ }+ std::thread::sleep(Duration::from_millis(400));+ let _ = fs::write(loss_file(), "0");+ let _ = fs::write(key_loss_file(), "0");++ Some(E2eFixture { _bag: bag, stdin, rx, pid, wid })+}++// ββ DOM registration oracle (trycua test apps serve an event log on 6769) βββββ++const APP_API: &str = "http://127.0.0.1:6769";++fn http_reset() {+ let _ = Command::new("curl.exe")+ .args(["-s", "-m", "3", "-X", "POST", &format!("{APP_API}/reset")])+ .stdout(Stdio::null()).stderr(Stdio::null()).status();+}+/// Returns the `/events` body, or None if the app isn't serving the API.+fn http_events() -> Option<String> {+ let out = Command::new("curl.exe")+ .args(["-s", "-m", "3", &format!("{APP_API}/events")])+ .output().ok()?;+ if !out.status.success() { return None; }+ let body = String::from_utf8_lossy(&out.stdout).into_owned();+ // Reachable iff it looks like the JSON array the app returns.+ if body.trim_start().starts_with('[') { Some(body) } else { None }+}+/// True if the event log recorded at least one DOM event since the last reset+/// (i.e. the action actually reached the page). None if the app has no API.+fn registered_since_reset() -> Option<bool> {+ http_events().map(|b| b.trim() != "[]" && !b.trim().is_empty())+}++fn foreground_pid() -> u32 {+ use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId};+ unsafe {+ let h = GetForegroundWindow();+ let mut pid = 0u32;+ GetWindowThreadProcessId(h, Some(&mut pid));+ pid+ }+}++/// The reliable "no z-raise / no foreground steal" oracle, independent of the+/// machine's foreground-lock setting: after acting on a BACKGROUND target, the+/// target's process must never become the foreground window. Polls for ~1.2s+/// to also catch async self-reactivation (Chromium). `user_pid` is the window+/// we expect to keep the foreground (the focus-monitor-win sentinel launched+/// last); reported for diagnostics.+fn assert_target_stays_background<F: FnOnce()>(label: &str, target_pid: u32, action: F) {+ let user_pid = read_count(&focus_pid_file());+ let fg_before = foreground_pid();+ if fg_before == target_pid {+ eprintln!("[e2e] WARN {label}: target pid {target_pid} was already foreground before the action (setup couldn't background it)");+ }+ action();+ let deadline = Instant::now() + Duration::from_millis(1200);+ let mut stole = false;+ let mut last_fg = fg_before;+ while Instant::now() < deadline {+ last_fg = foreground_pid();+ if last_fg == target_pid { stole = true; break; }+ std::thread::sleep(Duration::from_millis(80));+ }+ assert!(!stole,+ "{label}: target pid {target_pid} BECAME the foreground window β z-raise / foreground steal. \+ (expected user pid {user_pid}, fg_before={fg_before})");+ println!("β {label}: target pid {target_pid} stayed background (foreground pid={last_fg}, user={user_pid})");+}++/// Shared body: default-mode left click into a webview target.+fn webview_click_case(label: &str, exe: PathBuf) {+ let mut fx = match setup(&exe, "") { Some(f) => f, None => return };+ let (pid, wid) = (fx.pid, fx.wid);+ let snap = call(&mut fx.stdin, &fx.rx, 20, "get_window_state",+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));+ let elem = find_idx_containing(&snap, "button").or_else(|| find_idx_containing(&snap, "click"));++ http_reset();+ let mut last = serde_json::Value::Null;+ assert_target_stays_background(label, pid, || {+ let args = match elem {+ Some(idx) => serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx}),+ None => serde_json::json!({"pid": pid as i64, "window_id": wid, "x": 200, "y": 200}),+ };+ last = call(&mut fx.stdin, &fx.rx, 21, "click", args);+ });+ assert!(!is_error(&last), "{label}: default-mode click errored: {}", result_text(&last));+ assert!(!result_text(&last).contains("background_unavailable"),+ "{label}: click should not need dispatch:foreground, got {:?}", result_text(&last));+ // Delivery is confirmed by the driver's own β result (UIA Invoke fires the+ // element's default action / DOM `click`; PostMessage/injection deliver the+ // button events). The /events pointer-log is reported for info only β it+ // does NOT capture accessibility-driven UIA Invoke (which raises `click`,+ // not `pointerdown`), so a `[]` there is expected for the UIA path and is+ // not a failure. The hard invariant of this suite is no-foreground-steal.+ std::thread::sleep(Duration::from_millis(300));+ let reg = registered_since_reset().map(|b| b.to_string()).unwrap_or_else(|| "n/a".into());+ println!("{label}: delivered={:?} pointer-events-logged={reg}", result_text(&last));+}++// ββ tests βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++#[test]+#[ignore]+fn e2e_electron_background_click_no_z_raise() {+ let Some(exe) = electron_exe() else { eprintln!("[e2e] no electron app β skipping"); return; };+ webview_click_case("electron click (default dispatch)", exe);+}++#[test]+#[ignore]+fn e2e_tauri_background_click_no_z_raise() {+ let Some(exe) = tauri_exe() else { eprintln!("[e2e] no tauri app β skipping"); return; };+ webview_click_case("tauri click (default dispatch)", exe);+}++#[test]+#[ignore]+fn e2e_win32_notepad_background_click_no_z_raise() {+ let mut fx = match setup(Path::new(r"C:\Windows\System32\notepad.exe"), "") { Some(f) => f, None => return };+ let (pid, wid) = (fx.pid, fx.wid);+ let mut last = serde_json::Value::Null;+ assert_target_stays_background("notepad click (default dispatch)", pid, || {+ last = call(&mut fx.stdin, &fx.rx, 21, "click",+ serde_json::json!({"pid": pid as i64, "window_id": wid, "x": 120, "y": 120}));+ });+ assert!(!is_error(&last), "notepad click errored: {}", result_text(&last));+ println!("notepad click result: {:?}", result_text(&last));+}++/// Electron right-click (pen-barrel injection): no raise. Penβright promotion+/// is app-dependent, so landing is best-effort; the hard invariant is no+/// z-raise + no background_unavailable.+#[test]+#[ignore]+fn e2e_electron_background_right_click_no_z_raise() {+ let Some(exe) = electron_exe() else { eprintln!("[e2e] no electron app β skipping"); return; };+ let mut fx = match setup(&exe, "") { Some(f) => f, None => return };+ let (pid, wid) = (fx.pid, fx.wid);+ let mut last = serde_json::Value::Null;+ assert_target_stays_background("electron right-click (default dispatch)", pid, || {+ last = call(&mut fx.stdin, &fx.rx, 21, "click",+ serde_json::json!({"pid": pid as i64, "window_id": wid, "x": 200, "y": 200, "button": "right"}));+ });+ println!("electron right-click result: {:?}", result_text(&last));+}++/// Electron TEXT typing: focus a field, then type β must register the text in+/// the DOM AND never steal foreground / move the cursor. This is the "type into+/// any window" half of the mission for plain text (WM_CHAR path, no foreground).+#[test]+#[ignore]+fn e2e_electron_background_type_text_no_z_raise() {+ let Some(exe) = electron_exe() else { eprintln!("[e2e] no electron app β skipping"); return; };+ let mut fx = match setup(&exe, "") { Some(f) => f, None => return };+ let (pid, wid) = (fx.pid, fx.wid);++ // Focus a text field if the page exposes one (so the chars have a sink).+ let snap = call(&mut fx.stdin, &fx.rx, 20, "get_window_state",+ serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}));+ let field = find_idx_containing(&snap, "edit")+ .or_else(|| find_idx_containing(&snap, "text"))+ .or_else(|| find_idx_containing(&snap, "input"));+ if let Some(idx) = field {+ let _ = call(&mut fx.stdin, &fx.rx, 21, "click",+ serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx}));+ std::thread::sleep(Duration::from_millis(200));+ }++ http_reset();+ let mut last = serde_json::Value::Null;+ assert_target_stays_background("electron type_text (default dispatch)", pid, || {+ last = call(&mut fx.stdin, &fx.rx, 22, "type_text",+ serde_json::json!({"pid": pid as i64, "window_id": wid, "text": "cuatest"}));+ });+ assert!(!is_error(&last), "type_text errored: {}", result_text(&last));+ std::thread::sleep(Duration::from_millis(300));+ if let Some(reg) = registered_since_reset() {+ // Soft: typing needs a focused sink; if the page had no focusable field+ // we can't fault the input path. Report either way.+ println!("electron type_text: registered={reg} result={:?}", result_text(&last));+ } else {+ println!("electron type_text result: {:?}", result_text(&last));+ }+}++/// Electron key-combo (Ctrl+A): the dropped-PostMessage keyboard path now uses+/// cloaked focus + SendInput. Hard invariant: no z-raise.+#[test]+#[ignore]+fn e2e_electron_background_keycombo_no_z_raise() {+ let Some(exe) = electron_exe() else { eprintln!("[e2e] no electron app β skipping"); return; };+ let mut fx = match setup(&exe, "") { Some(f) => f, None => return };+ let (pid, wid) = (fx.pid, fx.wid);+ let mut last = serde_json::Value::Null;+ assert_target_stays_background("electron Ctrl+A (default dispatch)", pid, || {+ last = call(&mut fx.stdin, &fx.rx, 21, "press_key",+ serde_json::json!({"pid": pid as i64, "window_id": wid, "key": "a", "modifiers": ["control"]}));+ });+ // Capability-first: the combo must be DELIVERED, not refused. (UX is+ // best-effort: the cloaked focus restores foreground, so the no-steal+ // assertion above still holds post-action.)+ assert!(!is_error(&last) && !result_text(&last).contains("background_unavailable"),+ "Ctrl+A must be delivered (capability over UX), got: {:?}", result_text(&last));+ println!("electron Ctrl+A delivered: {:?}", result_text(&last));+}diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/motion.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/motion.rsindex 59d9586730..eba976ca50 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/motion.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/motion.rs@@ -31,6 +31,9 @@ pub struct MotionConfig {pub min_start_speed: f64,/// Minimum cursor speed at end of glide (deceleration floor), pts/sec.pub min_end_speed: f64,+ /// Minimum turning radius of the Dubins glide path, in points. Smaller =+ /// tighter curves. Matches the Swift reference default of 80.+ pub turn_radius: f64,}impl Default for MotionConfig {@@ -48,11 +51,13 @@ impl Default for MotionConfig {peak_speed: 900.0,min_start_speed: 300.0,min_end_speed: 200.0,+ turn_radius: 80.0,}}}impl MotionConfig {+ #[allow(clippy::too_many_arguments)]pub fn with_overrides(&self,start_handle: Option<f64>,@@ -64,6 +69,7 @@ impl MotionConfig {dwell_after_click_ms: Option<f64>,idle_hide_ms: Option<f64>,press_duration_ms: Option<f64>,+ turn_radius: Option<f64>,) -> Self {fn clamp(v: f64, lo: f64, hi: f64) -> f64 { v.clamp(lo, hi) }Self {@@ -79,6 +85,7 @@ impl MotionConfig {peak_speed: self.peak_speed,min_start_speed: self.min_start_speed,min_end_speed: self.min_end_speed,+ turn_radius: clamp(turn_radius.unwrap_or(self.turn_radius), 1.0, 1000.0),}}}diff --git a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rsindex 82fb78cd78..3d3579a054 100644--- a/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs+++ b/libs/cua-driver/rust/crates/cursor-overlay/src/render_state.rs@@ -133,7 +133,8 @@ impl RenderStateCore {let mut fire_arrival = false;if let Some(ref p) = self.path {- let path_frac = (self.dist / p.length.max(1.0)).clamp(0.0, 1.0);+ let path_len = p.length.max(1.0);+ let path_frac = (self.dist / path_len).clamp(0.0, 1.0);let profile =16.0 * path_frac * path_frac * (1.0 - path_frac) * (1.0 - path_frac);let floor = if path_frac < 0.5 {@@ -141,19 +142,37 @@ impl RenderStateCore {} else {self.motion.min_end_speed};- let speed = (floor + (self.motion.peak_speed - floor) * profile).max(floor);+ let speed_based = (floor + (self.motion.peak_speed - floor) * profile).max(floor);+ // Fixed-duration override: when `glide_duration_ms > 0` the move+ // takes exactly that long regardless of distance, so an orchestrator+ // can lock glides to a known cadence. `0` (the default) keeps the+ // speed-based timing untouched. Shared verbatim with the macOS+ // reference path (`tick_swift_constants`) β no platform drift.+ let speed = if self.motion.glide_duration_ms > 0.0 {+ path_len / (self.motion.glide_duration_ms / 1000.0)+ } else {+ speed_based+ };self.dist += speed * dt;- let path_len = p.length.max(1.0);if self.dist >= path_len {let end = p.sample(path_len);let end_heading = p.end_visual_heading;let vh = end.heading;+ // In fixed-duration mode the constant speed can be large; base+ // the settle impulse on the normal end-floor so the landing+ // stays as crisp as a speed-based glide instead of overshooting+ // proportionally to a short duration.+ let impulse = if self.motion.glide_duration_ms > 0.0 {+ self.motion.min_end_speed+ } else {+ speed+ };self.spring = Some(Spring {ox: 0.0,oy: 0.0,- vx: speed * 0.5 * vh.cos(),- vy: speed * 0.5 * vh.sin(),+ vx: impulse * 0.5 * vh.cos(),+ vy: impulse * 0.5 * vh.sin(),});self.spring_tgt = Some((end.x, end.y, end_heading));self.pos = (end.x, end.y);@@ -164,9 +183,11 @@ impl RenderStateCore {} else {let s: PathState = p.sample(self.dist);self.pos = (s.x, s.y);- let desired = s.heading + std::f64::consts::PI;- let max_step = 14.0 * dt;- self.heading = crate::util::rotate_toward(self.heading, desired, max_step);+ // Point the arrow exactly along the path tangent (the renderer+ // adds Ο, so we store tangent+Ο). Assigned directly rather than+ // rate-limited toward it, so the tip actually tracks the+ // trajectory instead of lagging behind on fast/short glides.+ self.heading = s.heading + std::f64::consts::PI;}} else if let Some(mut s) = self.spring {if let Some((tx, ty, th)) = self.spring_tgt {@@ -228,7 +249,17 @@ impl RenderStateCore {// Smootherstep speed profile (normalised: peak = 1.0).let profile = (30.0 * u * u * (1.0 - u) * (1.0 - u)) / 1.875;let floor_speed = if u < 0.5 { MIN_START_SPEED } else { MIN_END_SPEED };- let current_speed = floor_speed + (PEAK_SPEED - floor_speed) * profile;+ let speed_based = floor_speed + (PEAK_SPEED - floor_speed) * profile;+ // Fixed-duration override: when `glide_duration_ms > 0` the move+ // takes exactly that long regardless of distance, so an orchestrator+ // can lock glides to a known cadence. `0` (the default) keeps the+ // speed-based timing untouched. Shared verbatim with the+ // Windows/Linux path (`tick_motion`) β no platform drift.+ let current_speed = if self.motion.glide_duration_ms > 0.0 {+ path_len / (self.motion.glide_duration_ms / 1000.0)+ } else {+ speed_based+ };self.dist += current_speed * dt;if self.dist >= path_len {@@ -236,11 +267,20 @@ impl RenderStateCore {let end = p.sample(path_len);let end_heading = p.end_visual_heading;let vh = end.heading;+ // In fixed-duration mode the constant speed can be large; base+ // the settle impulse on the normal end-floor so the landing+ // stays as crisp as a speed-based glide instead of overshooting+ // proportionally to a short duration.+ let impulse = if self.motion.glide_duration_ms > 0.0 {+ MIN_END_SPEED+ } else {+ current_speed+ };self.spring = Some(Spring {ox: 0.0,oy: 0.0,- vx: current_speed * SPRING_OVERSHOOT * vh.cos(),- vy: current_speed * SPRING_OVERSHOOT * vh.sin(),+ vx: impulse * SPRING_OVERSHOOT * vh.cos(),+ vy: impulse * SPRING_OVERSHOOT * vh.sin(),});self.spring_tgt = Some((end.x, end.y, end_heading));self.pos = (end.x, end.y);@@ -251,10 +291,10 @@ impl RenderStateCore {} else {let s: PathState = p.sample(self.dist);self.pos = (s.x, s.y);- // Smooth heading rotation toward motion heading.- let desired = s.heading + std::f64::consts::PI;- let max_step = 14.0 * dt;- self.heading = crate::util::rotate_toward(self.heading, desired, max_step);+ // Point the arrow exactly along the path tangent (renderer adds+ // Ο, so store tangent+Ο). Direct assignment, not rate-limited, so+ // the tip tracks the trajectory instead of lagging on fast moves.+ self.heading = s.heading + std::f64::consts::PI;}} else if let Some(mut s) = self.spring {if let Some((tx, ty, th)) = self.spring_tgt {@@ -347,7 +387,7 @@ impl RenderStateCore {// tx = clickPoint.x + cos(endAngle) * clickOffset// ty = clickPoint.y + sin(endAngle) * clickOffsetconst CLICK_OFFSET: f64 = 16.0;- const TURN_RADIUS: f64 = 80.0;+ let turn_radius = self.motion.turn_radius;let tx = x + end_heading_radians.cos() * CLICK_OFFSET;let ty = y + end_heading_radians.sin() * CLICK_OFFSET;@@ -367,7 +407,7 @@ impl RenderStateCore {ty,th1,end_heading_radians,- TURN_RADIUS,+ turn_radius,);self.path = Some(plan);self.dist = 0.0;@@ -759,3 +799,52 @@ pub fn draw_default_arrow(None,);}++#[cfg(test)]+mod glide_duration_tests {+ use super::*;+ use crate::{CursorConfig, PathPlanner};++ /// Run a glide of `dist_pts` to completion and return how many seconds it+ /// took. `tick` selects the platform path: `false` = `tick_motion`+ /// (Windows/Linux), `true` = `tick_swift_constants` (macOS reference).+ fn arrival_secs(glide_ms: f64, dist_pts: f64, swift: bool) -> f64 {+ let mut core = RenderStateCore::new(CursorConfig::default());+ core.motion.glide_duration_ms = glide_ms;+ core.motion.idle_hide_ms = 0.0;+ core.pos = (0.0, 0.0);+ // Aligned headings β an effectively straight path of length ~dist_pts.+ core.path = Some(PathPlanner::plan(0.0, 0.0, 0.0, dist_pts, 0.0, 0.0, 0.0, 80.0));+ core.dist = 0.0;+ let dt = 1.0 / 240.0;+ let mut t = 0.0;+ for _ in 0..200_000 {+ let arrived = if swift { core.tick_swift_constants(dt) } else { core.tick_motion(dt) };+ t += dt;+ if arrived { break; }+ }+ t+ }++ #[test]+ fn fixed_duration_is_distance_independent_on_both_paths() {+ for swift in [false, true] {+ let short = arrival_secs(300.0, 120.0, swift);+ let long = arrival_secs(300.0, 1400.0, swift);+ // Both land in ~300ms regardless of distance (within a few ticks).+ assert!((short - 0.3).abs() < 0.05, "swift={swift} short={short}");+ assert!((long - 0.3).abs() < 0.05, "swift={swift} long={long}");+ }+ }++ #[test]+ fn zero_keeps_speed_based_timing() {+ // glide_duration_ms == 0 (the default) β longer paths take longer, on+ // both platform paths, exactly as before this field was implemented.+ for swift in [false, true] {+ let short = arrival_secs(0.0, 120.0, swift);+ let long = arrival_secs(0.0, 1400.0, swift);+ assert!(long > short + 0.2, "swift={swift} short={short} long={long}");+ }+ }+}diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rsindex 58c9e99f8d..c927f0144a 100644--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/cursor_tools.rs@@ -123,7 +123,7 @@ fn motion_def() -> &'static ToolDef {- arc_size: perpendicular deflection as fraction of path length [0,1]. Default 0.25\n\- arc_flow: asymmetry [-1,1]; positive bulges toward destination. Default 0.0\n\- spring: settle damping [0.3,1.0]; 1.0=no overshoot. Default 0.72\n\- - glide_duration_ms: flight duration per move [50,5000]. Default 160\n\+ - glide_duration_ms: fixed flight duration per move [50,5000]; omit for speed-based (the default)\n\- dwell_after_click_ms: pause after click ripple [0,5000]. Default 80\n\- idle_hide_ms: auto-hide delay [0,60000]; 0=never. Default 20000".into(),input_schema: serde_json::json!({@@ -159,7 +159,7 @@ fn motion_def() -> &'static ToolDef {"type": "number","minimum": 50,"maximum": 5000,- "description": "Flight duration per move in ms. Default 160."+ "description": "Fixed flight duration per move in ms; omit for speed-based timing (the default)."},"dwell_after_click_ms": {"type": "number",@@ -172,6 +172,12 @@ fn motion_def() -> &'static ToolDef {"minimum": 0,"maximum": 60000,"description": "Auto-hide delay in ms. 0 = never hide. Default 20000."+ },+ "turn_radius": {+ "type": "number",+ "minimum": 1,+ "maximum": 1000,+ "description": "Minimum turning radius of the glide path in points; smaller = tighter curves. Default 80."}},"additionalProperties": false@@ -226,7 +232,8 @@ impl Tool for SetAgentCursorMotionTool {|| args.get("spring").is_some()|| args.get("glide_duration_ms").is_some()|| args.get("dwell_after_click_ms").is_some()- || args.get("idle_hide_ms").is_some();+ || args.get("idle_hide_ms").is_some()+ || args.get("turn_radius").is_some();if motion_changed {// Read this cursor's current motion from the overlay, apply@@ -242,6 +249,7 @@ impl Tool for SetAgentCursorMotionTool {num(args.get("dwell_after_click_ms")),num(args.get("idle_hide_ms")),None, // press_duration_ms not exposed+ num(args.get("turn_radius")),);crate::cursor::overlay::send_command(cursor_id.clone(),diff --git a/libs/cua-driver/rust/crates/platform-windows/Cargo.toml b/libs/cua-driver/rust/crates/platform-windows/Cargo.tomlindex c40c8eccef..bd64e08e43 100644--- a/libs/cua-driver/rust/crates/platform-windows/Cargo.toml+++ b/libs/cua-driver/rust/crates/platform-windows/Cargo.toml@@ -32,6 +32,12 @@ windows = { version = "0.58", features = ["Win32_Graphics_Gdi","Win32_Graphics_Dwm","Win32_UI_Input_KeyboardAndMouse",+ # Pointer/touch injection (Track A background-actuation probe):+ # InitializeTouchInjection / InjectTouchInput / InjectSyntheticPointerInput+ # live in Win32_UI_Input_Pointer; POINTER_TYPE_INFO / HSYNTHETICPOINTERDEVICE+ # / CreateSyntheticPointerDevice live in Win32_UI_Controls.+ "Win32_UI_Input_Pointer",+ "Win32_UI_Controls","Win32_UI_HiDpi","Win32_System_LibraryLoader","Win32_Graphics_OpenGL",diff --git a/libs/cua-driver/rust/crates/platform-windows/examples/zdrop_probe.rs b/libs/cua-driver/rust/crates/platform-windows/examples/zdrop_probe.rsnew file mode 100644index 0000000000..8fbdef461c--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-windows/examples/zdrop_probe.rs@@ -0,0 +1,314 @@+//! Z-drop harness + Track A background-input probe.+//!+//! Measures the foreground "flash": when actuating input into a *background*+//! window, does the target window rise above the user's foreground window?+//!+//! The harness mirrors the `flash-repro` methodology referenced in+//! `uia/fg_bypass.rs`: capture the user's foreground window as `baseline`, then+//! poll the z-order at high frequency while an actuation runs on another thread.+//! A "z-drop" is any sample where `target` sits above `baseline` (or `baseline`+//! is no longer the foreground). 0 z-drops == no visible flash.+//!+//! Two actuators are compared against the SAME background target:+//! - control: SendInput with a brief SetForegroundWindow swap (the existing+//! flash path in `input/mouse.rs::send_click_synthesized`).+//! - inject : Track A β pointer/touch injection (InitializeTouchInjection ++//! InjectTouchInput), which RE showed has no foreground precondition+//! (NtUserInjectMouseInput gates only on per-process injection-enable, not+//! GetForegroundWindow β see docs/windows-background-input-re-plan.rs Β§4.4).+//!+//! Usage (run from an interactive desktop session, with a Chrome/WPF/etc window+//! open in the BACKGROUND and a different window focused in front):+//! cargo run -p platform-windows --example zdrop_probe -- [mode] [pid]+//! mode = both (default) | control | inject | list+//!+//! It prints the candidate windows, the baseline + target, a 2s countdown+//! (position your windows), then the z-drop numbers per actuator.+//!+//! WARNING: this performs a real left-click/tap at the target window's center.+//! Pick a target where a center click is harmless, or pass an explicit pid.++#[cfg(target_os = "windows")]+fn main() {+ probe::run();+}++#[cfg(not(target_os = "windows"))]+fn main() {+ eprintln!("zdrop_probe is Windows-only");+}++#[cfg(target_os = "windows")]+mod probe {+ use std::thread;+ use std::time::{Duration, Instant};++ use windows::Win32::Foundation::{BOOL, HANDLE, HWND, LPARAM, POINT, RECT, TRUE};+ use windows::Win32::UI::HiDpi::{+ SetProcessDpiAwarenessContext, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2,+ };+ use windows::Win32::UI::Input::Pointer::{+ InitializeTouchInjection, InjectTouchInput, POINTER_FLAG_DOWN, POINTER_FLAG_INCONTACT,+ POINTER_FLAG_INRANGE, POINTER_FLAG_UP, POINTER_INFO, POINTER_TOUCH_INFO,+ TOUCH_FEEDBACK_DEFAULT,+ };+ use windows::Win32::UI::Input::KeyboardAndMouse::{+ SendInput, INPUT, INPUT_0, INPUT_MOUSE, MOUSEINPUT, MOUSEEVENTF_LEFTDOWN,+ MOUSEEVENTF_LEFTUP,+ };+ use windows::Win32::UI::WindowsAndMessaging::{+ EnumWindows, GetClassNameW, GetCursorPos, GetForegroundWindow, GetWindow, GetWindowRect,+ GetWindowTextW, GetWindowThreadProcessId, GetTopWindow, IsWindowVisible, SetCursorPos,+ SetForegroundWindow, GW_HWNDNEXT, PT_TOUCH,+ };++ // ---- z-order helpers ----------------------------------------------------++ /// Walk the top-level z-order from the top; return true if `a` is above `b`.+ /// None if either isn't found in the walk.+ unsafe fn is_above(a: HWND, b: HWND) -> Option<bool> {+ let mut h = GetTopWindow(None).ok()?;+ loop {+ if h == a {+ return Some(true);+ }+ if h == b {+ return Some(false);+ }+ match GetWindow(h, GW_HWNDNEXT) {+ Ok(n) if !n.0.is_null() => h = n,+ _ => return None,+ }+ }+ }++ struct PollResult {+ samples: u64,+ target_above: u64,+ fg_not_baseline: u64,+ }++ /// Sample the z-order for `dur`, counting flashes of `target` over `baseline`.+ fn poll(baseline: usize, target: usize, dur: Duration) -> PollResult {+ let baseline = HWND(baseline as *mut _);+ let target = HWND(target as *mut _);+ let deadline = Instant::now() + dur;+ let mut r = PollResult { samples: 0, target_above: 0, fg_not_baseline: 0 };+ while Instant::now() < deadline {+ unsafe {+ r.samples += 1;+ if GetForegroundWindow() != baseline {+ r.fg_not_baseline += 1;+ }+ if is_above(target, baseline) == Some(true) {+ r.target_above += 1;+ }+ }+ thread::sleep(Duration::from_millis(4));+ }+ r+ }++ // ---- actuators ----------------------------------------------------------++ /// Control: the existing flash path β SetForegroundWindow swap + SendInput.+ fn actuate_sendinput_swap(target: usize, x: i32, y: i32) {+ let target = HWND(target as *mut _);+ unsafe {+ let prev_fg = GetForegroundWindow();+ let mut prev_cursor = POINT::default();+ let _ = GetCursorPos(&mut prev_cursor);+ let _ = SetForegroundWindow(target);+ thread::sleep(Duration::from_millis(8));+ let _ = SetCursorPos(x, y);+ let down = INPUT {+ r#type: INPUT_MOUSE,+ Anonymous: INPUT_0 {+ mi: MOUSEINPUT { dwFlags: MOUSEEVENTF_LEFTDOWN, ..Default::default() },+ },+ };+ let up = INPUT {+ r#type: INPUT_MOUSE,+ Anonymous: INPUT_0 {+ mi: MOUSEINPUT { dwFlags: MOUSEEVENTF_LEFTUP, ..Default::default() },+ },+ };+ let events = [down, up];+ SendInput(&events, std::mem::size_of::<INPUT>() as i32);+ thread::sleep(Duration::from_millis(40));+ let _ = SetCursorPos(prev_cursor.x, prev_cursor.y);+ if !prev_fg.0.is_null() && prev_fg != target {+ let _ = SetForegroundWindow(prev_fg);+ }+ }+ }++ /// Track A: coordinate-routed touch injection. No SetForegroundWindow.+ /// `hwndTarget` is left NULL so the system hit-tests by screen coordinate.+ fn actuate_touch_inject(x: i32, y: i32) -> Result<(), String> {+ unsafe {+ // Per-process injection enable (idempotent; ignore "already init").+ let _ = InitializeTouchInjection(1, TOUCH_FEEDBACK_DEFAULT);+ let mk = |flags| POINTER_TOUCH_INFO {+ pointerInfo: POINTER_INFO {+ pointerType: PT_TOUCH,+ pointerId: 0,+ pointerFlags: flags,+ sourceDevice: HANDLE::default(),+ hwndTarget: HWND::default(),+ ptPixelLocation: POINT { x, y },+ ..Default::default()+ },+ touchFlags: 0,+ touchMask: 0,+ rcContact: RECT { left: x - 2, top: y - 2, right: x + 2, bottom: y + 2 },+ rcContactRaw: RECT { left: x - 2, top: y - 2, right: x + 2, bottom: y + 2 },+ orientation: 0,+ pressure: 512,+ };+ let down = mk(POINTER_FLAG_DOWN | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT);+ InjectTouchInput(&[down]).map_err(|e| format!("InjectTouchInput(down): {e}"))?;+ thread::sleep(Duration::from_millis(30));+ let up = mk(POINTER_FLAG_UP);+ InjectTouchInput(&[up]).map_err(|e| format!("InjectTouchInput(up): {e}"))?;+ Ok(())+ }+ }++ // ---- window discovery ----------------------------------------------------++ unsafe extern "system" fn collect(hwnd: HWND, lparam: LPARAM) -> BOOL {+ let v = &mut *(lparam.0 as *mut Vec<HWND>);+ if IsWindowVisible(hwnd).as_bool() {+ v.push(hwnd);+ }+ TRUE+ }++ fn class_of(h: HWND) -> String {+ let mut buf = [0u16; 128];+ let n = unsafe { GetClassNameW(h, &mut buf) };+ String::from_utf16_lossy(&buf[..n.max(0) as usize])+ }+ fn title_of(h: HWND) -> String {+ let mut buf = [0u16; 256];+ let n = unsafe { GetWindowTextW(h, &mut buf) };+ String::from_utf16_lossy(&buf[..n.max(0) as usize])+ }+ fn pid_of(h: HWND) -> u32 {+ let mut pid = 0u32;+ unsafe { GetWindowThreadProcessId(h, Some(&mut pid)) };+ pid+ }++ fn center(h: HWND) -> Option<(i32, i32)> {+ let mut r = RECT::default();+ unsafe { GetWindowRect(h, &mut r).ok()? };+ if r.right <= r.left || r.bottom <= r.top {+ return None;+ }+ Some(((r.left + r.right) / 2, (r.top + r.bottom) / 2))+ }++ fn run_trial(name: &str, baseline: HWND, target: HWND, x: i32, y: i32, actuate: impl FnOnce()) {+ let b = baseline.0 as usize;+ let t = target.0 as usize;+ let poller = thread::spawn(move || poll(b, t, Duration::from_millis(1200)));+ thread::sleep(Duration::from_millis(120)); // let poller establish "baseline on top"+ let _ = (x, y);+ actuate();+ let r = poller.join().unwrap();+ let pct = |n: u64| if r.samples == 0 { 0.0 } else { 100.0 * n as f64 / r.samples as f64 };+ println!(+ " [{name:8}] samples={:4} target_above_baseline={:4} ({:5.1}%) fg!=baseline={:4} ({:5.1}%)",+ r.samples, r.target_above, pct(r.target_above), r.fg_not_baseline, pct(r.fg_not_baseline)+ );+ }++ pub fn run() {+ unsafe {+ let _ = SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2);+ }+ let args: Vec<String> = std::env::args().skip(1).collect();+ let mode = args.get(0).map(|s| s.as_str()).unwrap_or("both");+ let forced_pid: Option<u32> = args.get(1).and_then(|s| s.parse().ok());++ let mut wins: Vec<HWND> = Vec::new();+ unsafe {+ let _ = EnumWindows(Some(collect), LPARAM(&mut wins as *mut _ as isize));+ }+ let fg = unsafe { GetForegroundWindow() };++ // Candidate = visible, titled, not the current foreground, not shell.+ let preferred = ["chrome", "msedge", "firefox", "notepad", "wpf", "soffice"];+ let candidates: Vec<HWND> = wins+ .iter()+ .copied()+ .filter(|&h| h != fg && !title_of(h).is_empty())+ .filter(|&h| {+ let c = class_of(h).to_lowercase();+ !c.contains("progman") && !c.contains("workerw") && !c.contains("shell_traywnd")+ })+ .collect();++ if mode == "list" || candidates.is_empty() {+ println!("Foreground (baseline): pid={} class={:?} title={:?}", pid_of(fg), class_of(fg), title_of(fg));+ println!("Candidate background windows:");+ for h in &candidates {+ println!(" pid={:6} class={:24} title={:?}", pid_of(*h), class_of(*h), title_of(*h));+ }+ if candidates.is_empty() {+ eprintln!("\nNo background candidate windows. Open one and focus a different window.");+ }+ if mode == "list" {+ return;+ }+ }++ let target = match forced_pid {+ Some(pid) => candidates.iter().copied().find(|&h| pid_of(h) == pid),+ None => candidates+ .iter()+ .copied()+ .find(|&h| {+ let t = title_of(h).to_lowercase();+ let c = class_of(h).to_lowercase();+ preferred.iter().any(|p| t.contains(p) || c.contains(p))+ })+ .or_else(|| candidates.first().copied()),+ };+ let Some(target) = target else {+ eprintln!("No target window selected.");+ return;+ };++ let Some((x, y)) = center(target) else {+ eprintln!("Target has no usable rect.");+ return;+ };++ println!("baseline (user fg): pid={} title={:?}", pid_of(fg), title_of(fg));+ println!(+ "target (background): pid={} class={:?} title={:?} click@({x},{y})",+ pid_of(target), class_of(target), title_of(target)+ );+ println!("Position your windows now β actuating in 2s. (target should stay BEHIND baseline)");+ thread::sleep(Duration::from_secs(2));++ if mode == "both" || mode == "control" {+ run_trial("control", fg, target, x, y, || actuate_sendinput_swap(target.0 as usize, x, y));+ thread::sleep(Duration::from_millis(400));+ }+ if mode == "both" || mode == "inject" {+ // re-read baseline: control may have left fg elsewhere; refocus check+ let base2 = unsafe { GetForegroundWindow() };+ run_trial("inject", base2, target, x, y, || {+ if let Err(e) = actuate_touch_inject(x, y) {+ eprintln!(" touch-inject error: {e}");+ }+ });+ }+ println!("\nInterpretation: control should show a high target_above%/fg!=baseline% (the flash).");+ println!("Track A (inject) showing ~0% target_above == background input with no visible raise.");+ }+}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/dispatch.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/dispatch.rsindex ece145b905..7f4410357d 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/dispatch.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/dispatch.rs@@ -92,15 +92,17 @@ pub fn dispatch_schema() -> Value {"enum": ["background", "foreground", "auto"],"default": "background","description":- "Dispatch mode. 'background' (default) refuses to swap \- foreground; returns a structured background_unavailable error \- if PostMessage would be silently dropped for this event kind \- on this target. 'foreground' explicitly accepts a brief \- SetForegroundWindow swap (SendInput path) β required to drive \- Chromium-content or GTK-button widget targets reliably. \- 'auto' uses cua-driver's internal heuristics (silent fallback \- to SendInput on known-problematic targets); opt-in for \- callers that prefer the historical behavior."+ "Dispatch mode. 'background' (default) never swaps foreground: it \+ routes through UIA Invoke / PostMessage, and for targets that \+ silently drop posted clicks (Chromium/Electron content, GTK \+ buttons) it transparently falls back to coordinate-based pointer \+ injection β so a caller can just target the app and click without \+ knowing its internals, and the window is never raised. (A \+ background_unavailable error only surfaces for inputs injection \+ can't express, e.g. a right/middle click on such a target.) \+ 'foreground' explicitly accepts a brief SetForegroundWindow swap \+ (SendInput path). 'auto' uses cua-driver's historical heuristics \+ (silent SendInput fallback on known-problematic targets)."})}@@ -118,6 +120,17 @@ pub fn would_be_silently_dropped(hwnd: u64, kind: EventKind) -> bool {// Chromium's IME path, which DOES consume Win32 messages.return matches!(kind, MouseClick | MouseMove | MouseScroll | KeyCombo);}+ if is_wpf_target_window(hwnd) {+ // WPF ignores PostMessage mouse (its input manager drops mouse messages+ // unless the live system cursor is over the window β verified: posted+ // WM_MOUSE* raise no WPF events). It must be driven by coordinate-routed+ // system-queue input. We use a PERSISTENT synthetic touch digitizer+ // (see inject::TOUCH_DEV): WPF's stylus stack binds to the standing+ // device and consumes the contact as touch/stylus β promoting to mouse+ // internally, with NO OS cursor movement. WM_CHAR keystrokes still work,+ // so flag only the pointer-class events.+ return matches!(kind, MouseClick | MouseMove | MouseScroll);+ }if is_gtk_target_window(hwnd) {// Conservative flag for GTK: button widgets ignore PostMessage// clicks, drawing-area widgets accept them. We cannot distinguish@@ -165,6 +178,15 @@ pub fn is_vcl_target_window(hwnd: u64) -> bool {class.starts_with("SAL")}+/// Detect WPF top-level windows. WPF hosts its visual tree in an HWND whose+/// class is `HwndWrapper[<module>;;<guid>]`. WPF TextBoxes consume only real+/// keyboard input routed through WPF's input manager, so a posted `WM_CHAR`+/// (the `post_type_text` path) is silently dropped β type_text must instead+/// deliver genuine SendInput keystrokes (see `inject_text_cloaked`).+pub fn is_wpf_target_window(hwnd: u64) -> bool {+ read_class_name(hwnd).starts_with("HwndWrapper")+}+/// Detect GTK/GDK top-level windows.////// GTK 3 on Windows uses class `gdkWindowToplevel`; GTK 4 usesdiff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rsnew file mode 100644index 0000000000..fe562a236e--- /dev/null+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs@@ -0,0 +1,792 @@+//! Universal background mouse actuator: coordinate-routed pointer/touch+//! injection that delivers a click to whatever window sits under a screen+//! point **without** `SetForegroundWindow` and **without** moving the user's+//! mouse cursor.+//!+//! Why this exists: the PostMessage path (`mouse::post_click`) is invisible and+//! never raises, but Chromium/Electron/GTK/WPF content silently ignore posted+//! synthetic `WM_*BUTTON` messages β their input arrives through the *system+//! input queue*, not the per-window message queue. The historical fallback for+//! those was `send_click_synthesized` (SendInput + a `SetForegroundWindow`+//! swap), which is exactly the visible "flash" we want to eliminate.+//!+//! Touch injection routes by coordinate through the system input queue (so+//! Chromium et al. accept it; the OS promotes it to `WM_*BUTTON` for legacy+//! Win32 windows that don't consume `WM_POINTER`), and β per the RE in+//! `docs/windows-background-input-re-plan.md` Β§4.4 β the kernel injection path+//! (`NtUserInjectMouseInput`/`NtUserInjectTouchInput`) gates only on a+//! per-process injection-enable, NOT on the target being foreground. The one+//! residual is that a tap on an *inactive* top-level window can still trigger+//! click-activation; we contain that with [`ZorderGuard`], which DWM-cloaks the+//! (background) target for the duration of the tap so any transient raise is+//! invisible, then restores the user's foreground window and uncloaks.+//!+//! Scope: left-button taps (single/double/triple). Right/middle have no clean+//! touch mapping; callers fall back to their existing routing for those.++use anyhow::{bail, Result};+use core::ffi::c_void;+use std::sync::{Mutex, MutexGuard, TryLockError};+use std::thread::sleep;+use std::time::{Duration, Instant};++/// Serializes the cloaked-foreground SendInput operations (`inject_key_cloaked`,+/// `inject_text_cloaked`). Concurrent sessions must not interleave foreground+/// swaps + SendInput on the single shared system input queue, or keystrokes get+/// garbled and foreground restores race. Acquired with a hard 1s ceiling so a+/// stuck holder can never deadlock the others β after 1s, callers proceed+/// unserialized (degraded, but never hung).+static FG_SERIAL: Mutex<()> = Mutex::new(());++fn fg_serialize() -> Option<MutexGuard<'static, ()>> {+ let deadline = Instant::now() + Duration::from_secs(1);+ loop {+ match FG_SERIAL.try_lock() {+ Ok(g) => return Some(g),+ Err(TryLockError::Poisoned(p)) => return Some(p.into_inner()),+ Err(TryLockError::WouldBlock) => {+ if Instant::now() >= deadline {+ return None; // auto-expire: proceed without the lock+ }+ sleep(Duration::from_millis(20));+ }+ }+ }+}++use windows::Win32::Foundation::{BOOL, FALSE, HANDLE, HWND, POINT, RECT, TRUE};+use windows::Win32::Graphics::Dwm::{DwmSetWindowAttribute, DWMWA_CLOAK};+use windows::Win32::UI::Controls::{+ CreateSyntheticPointerDevice, DestroySyntheticPointerDevice, HSYNTHETICPOINTERDEVICE,+ POINTER_FEEDBACK_DEFAULT, POINTER_TYPE_INFO, POINTER_TYPE_INFO_0,+};+use windows::Win32::UI::Input::Pointer::{+ InjectSyntheticPointerInput, POINTER_FLAG_DOWN, POINTER_FLAG_INCONTACT, POINTER_FLAG_INRANGE,+ POINTER_FLAG_UP, POINTER_FLAG_UPDATE, POINTER_INFO, POINTER_PEN_INFO, POINTER_TOUCH_INFO,+};+use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId};+use windows::Win32::UI::Input::KeyboardAndMouse::{+ SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYEVENTF_KEYUP, KEYEVENTF_UNICODE,+ VIRTUAL_KEY,+};+use windows::Win32::UI::WindowsAndMessaging::{+ GetAncestor, GetCursorPos, GetForegroundWindow, GetWindowLongPtrW, GetWindowThreadProcessId,+ IsWindow, SetCursorPos, SetForegroundWindow, SetWindowLongPtrW, SetWindowPos,+ SystemParametersInfoW, GA_ROOT, GWL_EXSTYLE, HWND_NOTOPMOST, HWND_TOP, HWND_TOPMOST, PT_PEN,+ PT_TOUCH, SPI_GETFOREGROUNDLOCKTIMEOUT, SPI_SETFOREGROUNDLOCKTIMEOUT, SWP_NOACTIVATE, SWP_NOMOVE,+ SWP_NOSIZE, SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS, WS_EX_NOACTIVATE,+};++/// Bring `target` to the foreground using the AttachThreadInput trick, which+/// inherits the current foreground thread's FG-lock token so the swap is+/// honored even on a foreground-locked session without UIAccess (mirrors the+/// `bring_to_front` tool). Single attach, no retry loop β bounded. Returns+/// whether `target` actually became foreground.+unsafe fn force_foreground_attached(target: HWND) -> bool {+ let cur = GetForegroundWindow();+ if cur == target {+ return true;+ }+ let my_tid = GetCurrentThreadId();+ let mut pid = 0u32;+ let cur_tid = GetWindowThreadProcessId(cur, Some(&mut pid));+ let attached = cur_tid != 0 && cur_tid != my_tid;+ if attached {+ let _ = AttachThreadInput(my_tid, cur_tid, true);+ }+ let _ = SetForegroundWindow(target);+ if attached {+ let _ = AttachThreadInput(my_tid, cur_tid, false);+ }+ GetForegroundWindow() == target+}++/// RAII guard that momentarily drops the system foreground-lock timeout so a+/// non-UIAccess process can `SetForegroundWindow`, then restores the user's+/// original value on drop. The change is **in-memory only** β `fWinIni` is 0,+/// so it is NOT written to the user profile (no `SPIF_UPDATEINIFILE`) and never+/// persists past this guard. Required on machines whose foreground-lock is+/// maxed (`SPI_GETFOREGROUNDLOCKTIMEOUT` large), which otherwise denies the+/// raise an occluded WPF window needs.+struct ForegroundLockGuard {+ prev: u32,+ active: bool,+}++impl ForegroundLockGuard {+ unsafe fn disable() -> Self {+ let flags = SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0);+ let mut prev: u32 = 0;+ let got = SystemParametersInfoW(+ SPI_GETFOREGROUNDLOCKTIMEOUT, 0,+ Some(&mut prev as *mut _ as *mut c_void), flags,+ )+ .is_ok();+ if got && prev != 0 {+ // value goes in pvParam for this action; 0 = no lock.+ let _ = SystemParametersInfoW(SPI_SETFOREGROUNDLOCKTIMEOUT, 0, None, flags);+ }+ Self { prev, active: got && prev != 0 }+ }+}++impl Drop for ForegroundLockGuard {+ fn drop(&mut self) {+ if self.active {+ unsafe {+ let _ = SystemParametersInfoW(+ SPI_SETFOREGROUNDLOCKTIMEOUT, 0,+ Some(self.prev as usize as *mut c_void),+ SYSTEM_PARAMETERS_INFO_UPDATE_FLAGS(0),+ );+ }+ }+ }+}++/// Make this process the "last input event" provider so Windows' foreground+/// lock permits our `SetForegroundWindow`. A non-UIAccess process can normally+/// only set the foreground if it (or the current foreground) sent the last+/// input; injecting a synthetic, side-effect-free keystroke (a lone Ctrl tap β+/// no menu activation like Alt, no cursor movement like a mouse event) makes+/// us that provider for the moment that follows.+unsafe fn foreground_unlock_keypoke() {+ let mk = |up: bool| INPUT {+ r#type: INPUT_KEYBOARD,+ Anonymous: INPUT_0 {+ ki: KEYBDINPUT {+ wVk: VIRTUAL_KEY(0x11), // VK_CONTROL+ wScan: 0,+ dwFlags: if up { KEYEVENTF_KEYUP } else { Default::default() },+ time: 0,+ dwExtraInfo: 0,+ },+ },+ };+ let ev = [mk(false), mk(true)];+ SendInput(&ev, std::mem::size_of::<INPUT>() as i32);+}++/// Forcefully bring `target` to the foreground β beating the foreground lock+/// even from a non-UIAccess process β by combining the AttachThreadInput trick+/// with the synthetic-input unlock above. Used for WPF, which only processes+/// injected stylus while it is the active foreground window (so an occluded WPF+/// must be genuinely raised). Returns whether `target` became foreground.+unsafe fn force_foreground_hard(target: HWND) -> bool {+ if GetForegroundWindow() == target {+ return true;+ }+ let my_tid = GetCurrentThreadId();+ let cur = GetForegroundWindow();+ let mut pid = 0u32;+ let cur_tid = GetWindowThreadProcessId(cur, Some(&mut pid));+ let attached = cur_tid != 0 && cur_tid != my_tid;+ if attached {+ let _ = AttachThreadInput(my_tid, cur_tid, true);+ }+ foreground_unlock_keypoke();+ let _ = SetForegroundWindow(target);+ let _ = SetWindowPos(target, HWND_TOP, 0, 0, 0, 0, SWP_NOMOVE | SWP_NOSIZE);+ if attached {+ let _ = AttachThreadInput(my_tid, cur_tid, false);+ }+ GetForegroundWindow() == target+}++/// RAII guard that makes a specific target window **unable to become the+/// foreground/active window** for the duration of an actuation, by adding the+/// `WS_EX_NOACTIVATE` extended style to its top-level window.+///+/// Why this and not a global foreground-lock: our own injected/posted input+/// (or a UIA-Invoke) legitimizes the target's foreground claim, so even a+/// maxed `SPI_*FOREGROUNDLOCKTIMEOUT` won't stop the steal. `WS_EX_NOACTIVATE`+/// is categorical β Windows refuses to activate the window *at all* (clicks,+/// `SetForegroundWindow(self)` from WPF/XAML/Tauri handlers, mouse-activate) β+/// while the window still RECEIVES the click/key. It is per-window (no session+/// side effects) and reversed on drop. Covers the self-activation that the+/// EnableWindow/UWP bypass cannot (WPF `UIElement.Focus()`βSetForegroundWindow).+pub struct NoActivateGuard {+ // Store the handle as an integer so the guard is `Send` and can be held+ // across `.await` in the async tools.+ root_addr: isize,+ prev_exstyle: isize,+ applied: bool,+}++impl NoActivateGuard {+ /// Arm on the top-level (GA_ROOT) ancestor of `hwnd`.+ pub fn arm(hwnd: HWND) -> Self {+ unsafe {+ let root = {+ let r = GetAncestor(hwnd, GA_ROOT);+ if r.0.is_null() { hwnd } else { r }+ };+ let prev = GetWindowLongPtrW(root, GWL_EXSTYLE);+ let want = WS_EX_NOACTIVATE.0 as isize;+ // Apply WS_EX_NOACTIVATE if not already set (prev can be 0, so don't+ // gate on it β we just need to check the bit and set it if absent).+ let applied = (prev & want) == 0 && {+ SetWindowLongPtrW(root, GWL_EXSTYLE, prev | want);+ // Confirm it took (cross-process SetWindowLongPtr can be denied+ // by UIPI on higher-integrity targets).+ (GetWindowLongPtrW(root, GWL_EXSTYLE) & want) != 0+ };+ Self { root_addr: root.0 as isize, prev_exstyle: prev, applied }+ }+ }+}++impl Drop for NoActivateGuard {+ fn drop(&mut self) {+ if self.applied {+ unsafe {+ let _ = SetWindowLongPtrW(HWND(self.root_addr as *mut _), GWL_EXSTYLE, self.prev_exstyle);+ }+ }+ }+}++/// PEN_FLAG_BARREL (winuser.h) β pen barrel button held == secondary (right)+/// button. `penFlags` is a raw u32 in the bindings, so use the literal.+const PEN_FLAG_BARREL: u32 = 0x00000001;++const CLOAK_SIZE: u32 = std::mem::size_of::<BOOL>() as u32;++/// Restore the user's window to the top of the visible z-order WITHOUT+/// activating it. `SWP_NOACTIVATE` sends no `WM_ACTIVATE`/`WM_MOUSEACTIVATE`+/// to either window, so this can never block on a busy target's activation+/// handler (the deadlock that `AttachThreadInput` + `SetForegroundWindow`+/// risks against a webview that's mid-click). It is also not gated by the+/// foreground-lock. Best-effort, instant, hang-free.+unsafe fn restore_z_top(user_win: HWND) {+ let _ = SetWindowPos(+ user_win,+ HWND_TOP,+ 0,+ 0,+ 0,+ 0,+ SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE,+ );+}++/// Put `win` into / out of the always-on-top (topmost) band WITHOUT activating+/// it. `SWP_NOACTIVATE` means no focus/foreground change. The topmost band sits+/// above ALL normal windows β including an *active* occluder β which `HWND_TOP`+/// alone does not guarantee for a non-activated (esp. `WS_EX_NOACTIVATE`)+/// window. Used to make a blocked injection target win the coordinate hit-test.+unsafe fn set_topmost(win: HWND, on: bool) {+ let after = if on { HWND_TOPMOST } else { HWND_NOTOPMOST };+ let _ = SetWindowPos(win, after, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE);+}++unsafe fn set_cloak(h: HWND, on: bool) -> bool {+ let v: BOOL = if on { TRUE } else { FALSE };+ DwmSetWindowAttribute(h, DWMWA_CLOAK, &v as *const _ as *const c_void, CLOAK_SIZE).is_ok()+}++/// RAII guard that lands coordinate-routed injection on an occluded background+/// target without stealing focus.+///+/// Coordinate injection (pen/touch) is delivered to the TOP-MOST **visible**+/// window at the screen point β and a DWM-cloaked window is *excluded* from+/// hit-testing (verified: injection over a cloaked target lands on the occluder+/// instead). So we cannot hide the target; to drive it when it's blocked we+/// briefly raise it to the top of the z-order on `arm` β with `SWP_NOACTIVATE`,+/// so the user's window keeps focus/foreground (no activation, no input-queue+/// attach) β and on `Drop` restore the user's window to the top. The target is+/// visible on top only for the few milliseconds of the actuation.+struct ZorderGuard {+ prev_fg: HWND,+ target: HWND,+ raised: bool,+}++impl ZorderGuard {+ unsafe fn arm(target: HWND) -> Self {+ let prev_fg = GetForegroundWindow();+ // TODO: Check if target is actually occluded (WindowFromPoint over its+ // client rect) before raising, and preserve its original topmost state+ // (via GetWindowLongPtr/WS_EX_TOPMOST) so drop() can restore it instead+ // of unconditionally demoting. Current behavior: raise any non-foreground+ // target into topmost band (works for common case, but loses original z+ // and raises even when not occluded).+ let raised = !target.0.is_null() && target != prev_fg;+ if raised {+ set_topmost(target, true);+ }+ Self { prev_fg, target, raised }+ }+}++impl Drop for ZorderGuard {+ fn drop(&mut self) {+ unsafe {+ if self.raised {+ // Drop the target back out of the topmost band, then re-stack the+ // user's window on top (hang-free, no activation messages).+ set_topmost(self.target, false);+ if !self.prev_fg.0.is_null() && self.prev_fg != self.target {+ restore_z_top(self.prev_fg);+ }+ }+ }+ }+}++/// One downβup **pen** tap at screen `(sx, sy)`. When `barrel` is set the pen's+/// barrel button is held for the contact, which the system maps to a secondary+/// (right) click β both for `WM_POINTER`-aware apps (Chromium/WPF/UWP) and via+/// penβmouse promotion for legacy Win32. A fresh synthetic pen device is+/// created and destroyed per tap (right/middle clicks are rare).+fn pen_tap(sx: i32, sy: i32, barrel: bool) -> Result<()> {+ unsafe {+ let dev = CreateSyntheticPointerDevice(PT_PEN, 1, POINTER_FEEDBACK_DEFAULT)+ .map_err(|e| anyhow::anyhow!("CreateSyntheticPointerDevice(PEN): {e}"))?;+ let pen_flags = if barrel { PEN_FLAG_BARREL } else { 0 };+ let mk = |flags| POINTER_TYPE_INFO {+ r#type: PT_PEN,+ Anonymous: POINTER_TYPE_INFO_0 {+ penInfo: POINTER_PEN_INFO {+ pointerInfo: POINTER_INFO {+ pointerType: PT_PEN,+ pointerId: 0,+ pointerFlags: flags,+ sourceDevice: HANDLE::default(),+ hwndTarget: HWND::default(),+ ptPixelLocation: POINT { x: sx, y: sy },+ ..Default::default()+ },+ penFlags: pen_flags,+ penMask: 0,+ pressure: 512,+ rotation: 0,+ tiltX: 0,+ tiltY: 0,+ },+ },+ };+ let down = mk(POINTER_FLAG_DOWN | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT);+ let r1 = InjectSyntheticPointerInput(dev, &[down]);+ sleep(Duration::from_millis(25));+ let up = mk(POINTER_FLAG_UP);+ let r2 = InjectSyntheticPointerInput(dev, &[up]);+ let _ = DestroySyntheticPointerDevice(dev);+ r1.and(r2).map_err(|e| anyhow::anyhow!("InjectSyntheticPointerInput(pen): {e}"))?;+ }+ Ok(())+}++/// Inject a click at **screen** coordinates `(sx, sy)`, routed by the system to+/// whatever window is under that point β without a foreground swap and without+/// moving the user's cursor. The target is cloaked for the duration so any+/// click-activation raise stays invisible, then the user's foreground is+/// restored.+///+/// - `left` β synthetic-pen primary tap (proven path; promoted to a left click+/// for non-pointer-aware apps, and accepted directly by Chromium/WPF/UWP).+/// - `right` β synthetic-pen tap with the barrel button held (secondary click).+/// - `middle`β unsupported (no clean pointer mapping); returns `Err` so the+/// caller can fall back to its existing routing / structured error.+///+/// We use the same `CreateSyntheticPointerDevice`/`InjectSyntheticPointerInput`+/// path for both buttons β `InjectTouchInput` proved unreliable for left-clicks+/// on Chromium content (returned errors), whereas synthetic-pen injection lands+/// reliably and routes by coordinate with no foreground dependency.+pub fn inject_click_screen(target: u64, sx: i32, sy: i32, count: usize, button: &str) -> Result<()> {+ if target == 0 {+ bail!("inject_click_screen: null target window");+ }+ let target_h = HWND(target as *mut _);+ unsafe {+ if !IsWindow(target_h).as_bool() {+ bail!("inject_click_screen: invalid or stale target HWND");+ }+ }+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target) {+ // Higher-integrity target: injection into its queue is blocked too.+ bail!(msg);+ }++ let barrel = match button {+ "left" => false,+ "right" => true,+ other => bail!("background injection supports left/right buttons only (got {other:?})"),+ };++ // Make the target categorically non-activatable for the click (so neither+ // click-activation nor a self-SetForegroundWindow can steal foreground),+ // and hide/restore any residual z-order via the cloak/SWP guard.+ let _noact = NoActivateGuard::arm(target_h);+ let _guard = unsafe { ZorderGuard::arm(target_h) };+ let count = count.max(1);+ for i in 0..count {+ pen_tap(sx, sy, barrel)?;+ if i + 1 < count {+ sleep(Duration::from_millis(70));+ }+ }+ // _guard drops here: restore the user's foreground + uncloak target.+ Ok(())+}++/// One pen press-drag-release from screen `(sx0,sy0)` to `(sx1,sy1)`, with+/// `steps` interpolated in-contact UPDATE points between the down and the up.+/// A single synthetic pen device is created for the whole stroke. The barrel+/// button is held when `barrel` is set (secondary-button drag).+fn pen_drag(sx0: i32, sy0: i32, sx1: i32, sy1: i32, steps: usize, barrel: bool) -> Result<()> {+ unsafe {+ let dev = CreateSyntheticPointerDevice(PT_PEN, 1, POINTER_FEEDBACK_DEFAULT)+ .map_err(|e| anyhow::anyhow!("CreateSyntheticPointerDevice(PEN): {e}"))?;+ let pen_flags = if barrel { PEN_FLAG_BARREL } else { 0 };+ let mk = |flags, x: i32, y: i32| POINTER_TYPE_INFO {+ r#type: PT_PEN,+ Anonymous: POINTER_TYPE_INFO_0 {+ penInfo: POINTER_PEN_INFO {+ pointerInfo: POINTER_INFO {+ pointerType: PT_PEN,+ pointerId: 0,+ pointerFlags: flags,+ sourceDevice: HANDLE::default(),+ hwndTarget: HWND::default(),+ ptPixelLocation: POINT { x, y },+ ..Default::default()+ },+ penFlags: pen_flags,+ penMask: 0,+ pressure: 512,+ rotation: 0,+ tiltX: 0,+ tiltY: 0,+ },+ },+ };+ // Press at the start.+ let down = mk(POINTER_FLAG_DOWN | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT, sx0, sy0);+ let mut res = InjectSyntheticPointerInput(dev, &[down]);+ // Interpolated in-contact moves so frameworks that gate drag-tracking on+ // motion (rather than a single downβup) see a continuous stroke.+ let steps = steps.max(1);+ for i in 1..=steps {+ sleep(Duration::from_millis(8));+ let t = i as f64 / steps as f64;+ let x = sx0 + ((sx1 - sx0) as f64 * t).round() as i32;+ let y = sy0 + ((sy1 - sy0) as f64 * t).round() as i32;+ let mv = mk(POINTER_FLAG_UPDATE | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT, x, y);+ res = res.and(InjectSyntheticPointerInput(dev, &[mv]));+ }+ // Release at the end.+ sleep(Duration::from_millis(8));+ let up = mk(POINTER_FLAG_UP, sx1, sy1);+ res = res.and(InjectSyntheticPointerInput(dev, &[up]));+ let _ = DestroySyntheticPointerDevice(dev);+ res.map_err(|e| anyhow::anyhow!("InjectSyntheticPointerInput(pen drag): {e}"))?;+ }+ Ok(())+}++/// A **persistent** synthetic touch digitizer, created once and never+/// destroyed. This is load-bearing: a transient (per-stroke) device is gone+/// before WPF's WISP stylus stack can bind to it, so the OS falls back to+/// legacy touchβmouse promotion β which drags the user's cursor to the+/// contact. A *standing* digitizer is enumerated as a real tablet, so WPF (and+/// other stylus/pointer-aware frameworks) consume the contact as touch/stylus+/// and promote it to mouse INTERNALLY, without the OS moving the system cursor.+static TOUCH_DEV: Mutex<isize> = Mutex::new(0);++/// One **touch** press-drag-release from screen `(sx0,sy0)` to `(sx1,sy1)` with+/// `steps` interpolated in-contact moves, via the persistent [`TOUCH_DEV`].+/// Unlike a pen (an absolute *cursor* device β injecting one drags the user's+/// mouse pointer along), a touch contact from a standing digitizer is consumed+/// as touch/stylus and does NOT move the user's cursor. Serialized on the+/// single shared device (one stroke at a time across all sessions).+fn touch_drag(sx0: i32, sy0: i32, sx1: i32, sy1: i32, steps: usize) -> Result<()> {+ let mut dev_guard = TOUCH_DEV.lock().unwrap_or_else(|e| e.into_inner());+ unsafe {+ // A non-pointer-aware window (WPF) makes the OS promote the PRIMARY touch+ // contact to a mouse event, which drags the system cursor to the contact+ // β and the OS gates delivery on the cursor actually reaching it, so the+ // move can't be prevented from a background process (pinning/clipping the+ // cursor just drops the input). What we CAN do is snap the cursor back to+ // exactly where the user left it the instant the stroke ends, so the net+ // displacement is zero and (with a fast, few-step stroke) the excursion+ // is a brief flick rather than a sustained drag. Pointer-aware targets+ // (Chromium) never promote, so the cursor never moves and this restore is+ // a harmless no-op.+ let mut cpos = POINT::default();+ let have_cpos = GetCursorPos(&mut cpos).is_ok();+ let dev = if *dev_guard != 0 {+ HSYNTHETICPOINTERDEVICE(*dev_guard as *mut c_void)+ } else {+ let d = CreateSyntheticPointerDevice(PT_TOUCH, 1, POINTER_FEEDBACK_DEFAULT)+ .map_err(|e| anyhow::anyhow!("CreateSyntheticPointerDevice(TOUCH): {e}"))?;+ *dev_guard = d.0 as isize;+ d+ };+ let mk = |flags, x: i32, y: i32| POINTER_TYPE_INFO {+ r#type: PT_TOUCH,+ Anonymous: POINTER_TYPE_INFO_0 {+ touchInfo: POINTER_TOUCH_INFO {+ pointerInfo: POINTER_INFO {+ pointerType: PT_TOUCH,+ pointerId: 0,+ pointerFlags: flags,+ sourceDevice: HANDLE::default(),+ hwndTarget: HWND::default(),+ ptPixelLocation: POINT { x, y },+ ..Default::default()+ },+ touchFlags: 0,+ touchMask: 0,+ rcContact: RECT { left: x - 2, top: y - 2, right: x + 2, bottom: y + 2 },+ rcContactRaw: RECT { left: x - 2, top: y - 2, right: x + 2, bottom: y + 2 },+ orientation: 0,+ pressure: 512,+ },+ },+ };+ // Fast stroke: line-tool canvases only need downβup (the segment is the+ // straight line between them), so a few in-contact frames with a tiny+ // dwell is plenty β and the shorter the stroke, the briefer the cursor+ // excursion before we snap it back.+ let down = mk(POINTER_FLAG_DOWN | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT, sx0, sy0);+ let mut res = InjectSyntheticPointerInput(dev, &[down]);+ let steps = steps.clamp(1, 3);+ for i in 1..=steps {+ sleep(Duration::from_millis(2));+ let t = i as f64 / steps as f64;+ let x = sx0 + ((sx1 - sx0) as f64 * t).round() as i32;+ let y = sy0 + ((sy1 - sy0) as f64 * t).round() as i32;+ let mv = mk(POINTER_FLAG_UPDATE | POINTER_FLAG_INRANGE | POINTER_FLAG_INCONTACT, x, y);+ res = res.and(InjectSyntheticPointerInput(dev, &[mv]));+ }+ sleep(Duration::from_millis(2));+ let up = mk(POINTER_FLAG_UP, sx1, sy1);+ res = res.and(InjectSyntheticPointerInput(dev, &[up]));+ // Snap the cursor back to where the user left it. The OS processes the+ // promoted mouse messages slightly after injection, so a single restore+ // right after the `up` can be overrun by that late move β settle briefly,+ // then restore, and restore once more to win the race. No-op for+ // pointer-aware targets (Chromium) that never moved the cursor.+ if have_cpos {+ let _ = SetCursorPos(cpos.x, cpos.y);+ sleep(Duration::from_millis(12));+ let _ = SetCursorPos(cpos.x, cpos.y);+ }+ // device intentionally NOT destroyed β see TOUCH_DEV.+ res.map_err(|e| anyhow::anyhow!("InjectSyntheticPointerInput(touch drag): {e}"))?;+ }+ Ok(())+}++/// Inject a press-drag-release at **screen** coordinates, routed by the system+/// to whatever window is under the path β without a foreground swap and without+/// moving the user's cursor. This is the background fallback for canvases whose+/// content (Chromium/WPF/GTK) silently drops a PostMessage drag: synthetic-pen+/// input arrives through the system input queue and is accepted directly+/// (Chromium/WPF) or promoted to mouse for legacy Win32. `left` β primary+/// stroke, `right` β barrel-held secondary stroke; the target is held+/// non-activatable + cloaked so any transient raise stays invisible.+pub fn inject_drag_screen(+ target: u64,+ sx0: i32,+ sy0: i32,+ sx1: i32,+ sy1: i32,+ steps: usize,+ button: &str,+) -> Result<()> {+ if target == 0 {+ bail!("inject_drag_screen: null target window");+ }+ let target_h = HWND(target as *mut _);+ unsafe {+ if !IsWindow(target_h).as_bool() {+ bail!("inject_drag_screen: invalid or stale target HWND");+ }+ }+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target) {+ bail!(msg);+ }+ let barrel = match button {+ "left" => false,+ "right" => true,+ other => bail!("background injection supports left/right buttons only (got {other:?})"),+ };+ let prev_fg = unsafe { GetForegroundWindow() };+ // Left drag β touch contact (coordinate-routed). Right/barrel drag has no+ // touch equivalent, so fall back to a pen (rare).+ let stroke = |()| if barrel { pen_drag(sx0, sy0, sx1, sy1, steps, true) } else { touch_drag(sx0, sy0, sx1, sy1, steps) };++ // WPF (Wisp input) only PROCESSES injected stylus while it is the ACTIVE+ // foreground window β raising it in z while it stays inactive is not enough+ // (verified by RE). So for WPF we must briefly activate it (a visible raise+ // + focus, which activeβforegroundβtopmost also un-occludes), inject, then+ // restore the user's foreground. Other coordinate-injection targets+ // (Chromium/GTK) are pointer-aware and process injection in the background,+ // so we hold them non-activatable and only raise them into the topmost band+ // to win the hit-test when occluded β no focus steal.+ let needs_active = crate::input::dispatch::is_wpf_target_window(target);+ if needs_active {+ // Break the no-raise contract for WPF: fully raise+activate it for the+ // brief moment of the stroke so its Wisp input stack processes the+ // injected stylus, then restore the user's window. The machine's+ // foreground-lock is dropped (in-memory only) for this window so the+ // raise is permitted, and restored immediately afterwards.+ let _lock = unsafe { ForegroundLockGuard::disable() };+ unsafe { force_foreground_hard(target_h); }+ let r = stroke(());+ unsafe {+ if !prev_fg.0.is_null() && prev_fg != target_h {+ force_foreground_hard(prev_fg);+ }+ }+ return r;+ }+ // Chromium/GTK: pointer-aware, process injection in the background β hold+ // non-activatable + raise into the topmost band to win the hit-test when+ // occluded, no focus steal.+ let r = {+ let _noact = NoActivateGuard::arm(target_h);+ let _guard = unsafe { ZorderGuard::arm(target_h) };+ stroke(())+ };+ unsafe {+ if !prev_fg.0.is_null() && prev_fg != target_h {+ force_foreground_attached(prev_fg);+ }+ }+ r+}++/// Send `key` (+ optional `modifiers`) to a **background** target via the+/// system input queue, with the target cloaked so the brief focus it needs+/// never shows as a visible raise.+///+/// Keyboard input β unlike mouse β has no coordinate routing: synthesized keys+/// go to the *focused* window of the foreground queue, so the target must hold+/// focus to receive a SendInput accelerator (Ctrl+S, Ctrl+A) that frameworks+/// detect via `GetKeyState`/`TranslateAccelerator`.+///+/// Capability-first contract: the keystroke MUST be delivered. We make a+/// best-effort to preserve the background UX (cloak the target so its brief+/// foreground stint is hidden, restore the user's foreground after), but we do+/// NOT abandon the action to keep the UX:+/// 1. Cloak the target (DWM) so any raise is invisible.+/// 2. Bring it foreground via the AttachThreadInput trick (beats the+/// foreground-lock even without UIAccess) and SendInput the combo, so+/// `GetKeyState` updates and the accelerator actually fires.+/// 3. If focus genuinely can't be obtained, fall back to PostMessage so the+/// key still reaches the window (best-effort; may miss GetKeyState-gated+/// accelerators, but never silently drops the action).+/// 4. Restore the user's foreground and uncloak.+pub fn inject_key_cloaked(target: u64, key: &str, modifiers: &[&str]) -> Result<()> {+ let target_h = HWND(target as *mut _);+ if target_h.0.is_null() {+ bail!("invalid target hwnd");+ }+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target) {+ bail!(msg);+ }++ let _serial = fg_serialize(); // one cloaked-foreground op at a time (1s ceiling)+ let prev_fg = unsafe { GetForegroundWindow() };+ let cloaked = unsafe { target_h != prev_fg && set_cloak(target_h, true) };+ let got_fg = unsafe { force_foreground_attached(target_h) };++ let result = if got_fg {+ // Target is foreground (cloaked): send_key_synthesized's own+ // SetForegroundWindow is a no-op success; SendInput updates GetKeyState+ // so the accelerator fires.+ crate::input::send_key_synthesized(target, key, modifiers)+ } else {+ // Couldn't focus the target even with the attach trick β deliver+ // best-effort via PostMessage rather than dropping the action.+ crate::input::post_key(target, key, modifiers)+ };++ unsafe {+ if !prev_fg.0.is_null() && prev_fg != target_h {+ force_foreground_attached(prev_fg);+ }+ if cloaked {+ let _ = set_cloak(target_h, false);+ }+ }+ result+}++/// Type `text` into a **background** target via real SendInput Unicode+/// keystrokes, cloaked so the brief focus is hidden, then restore foreground.+///+/// For targets that ignore a posted `WM_CHAR` (WPF, whose TextBox only consumes+/// real keyboard input routed through its own input manager), `post_type_text`+/// silently does nothing. This delivers genuine `KEYEVENTF_UNICODE` keystrokes+/// to the focused control while the target briefly (and invisibly) holds focus.+/// Capability-first: the text is delivered; the focus flicker is hidden and the+/// user's foreground restored. Caller should focus the field first (a prior+/// background click on it) so the keystrokes land in the right control.+pub fn inject_text_cloaked(target: u64, text: &str) -> Result<()> {+ let target_h = HWND(target as *mut _);+ if target_h.0.is_null() {+ bail!("invalid target hwnd");+ }+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target) {+ bail!(msg);+ }+ let _serial = fg_serialize(); // one cloaked-foreground op at a time (1s ceiling)+ let prev_fg = unsafe { GetForegroundWindow() };+ let cloaked = unsafe { target_h != prev_fg && set_cloak(target_h, true) };+ let got_fg = unsafe { force_foreground_attached(target_h) };++ let result = if got_fg {+ unsafe { send_unicode(text) }+ } else {+ crate::input::post_type_text(target, text)+ };++ unsafe {+ if !prev_fg.0.is_null() && prev_fg != target_h {+ force_foreground_attached(prev_fg);+ }+ if cloaked {+ let _ = set_cloak(target_h, false);+ }+ }+ result+}++fn key_unicode(unit: u16, up: bool) -> INPUT {+ let mut flags = KEYEVENTF_UNICODE;+ if up {+ flags |= KEYEVENTF_KEYUP;+ }+ INPUT {+ r#type: INPUT_KEYBOARD,+ Anonymous: INPUT_0 {+ ki: KEYBDINPUT {+ wVk: VIRTUAL_KEY(0),+ wScan: unit,+ dwFlags: flags,+ time: 0,+ dwExtraInfo: 0,+ },+ },+ }+}++unsafe fn send_unicode(text: &str) -> Result<()> {+ let mut ev: Vec<INPUT> = Vec::with_capacity(text.len() * 2);+ for u in text.encode_utf16() {+ ev.push(key_unicode(u, false));+ ev.push(key_unicode(u, true));+ }+ if ev.is_empty() {+ return Ok(());+ }+ let sent = SendInput(&ev, std::mem::size_of::<INPUT>() as i32);+ if sent as usize != ev.len() {+ bail!("SendInput typed only {sent} of {} key events", ev.len());+ }+ Ok(())+}diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rsindex bf7b1c3f49..d1bb662f88 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs@@ -12,7 +12,9 @@pub mod mouse;pub mod keyboard;pub mod dispatch;+pub mod inject;+pub use inject::{inject_click_screen, inject_key_cloaked, inject_text_cloaked, NoActivateGuard};pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized};pub use keyboard::{is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay,diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rsindex 8a7cbc2f47..ae447fab19 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs@@ -166,6 +166,62 @@ pub fn post_drag(Ok(())}+/// Press-drag-release via PostMessage, resolving the **deepest child** at the+/// drag-start screen point and posting in that child's client coordinates.+///+/// `post_drag` (above) posts to the top-level frame, so a child-windowed+/// control (a WinForms `Panel`, a Win32 child canvas, β¦) never sees the drag β+/// the frame gets messages over a region it doesn't own and ignores them. This+/// variant mirrors `post_click`: it hit-tests down to the deepest descendant+/// under the start point and targets that HWND for the whole gesture (a drag+/// stays within one control), with each point converted to the child's own+/// client space. Endpoints are given in **screen** coordinates.+pub fn post_drag_screen(+ root: u64,+ sx_from: i32,+ sy_from: i32,+ sx_to: i32,+ sy_to: i32,+ duration_ms: u64,+ steps: usize,+ button: &str,+) -> Result<()> {+ let root_hwnd = HWND(root as *mut _);+ let (target, c_from) = deepest_child(root_hwnd, POINT { x: sx_from, y: sy_from });+ let mut c_to = POINT { x: sx_to, y: sy_to };+ unsafe { let _ = ScreenToClient(target, &mut c_to); }+ if let Some(msg) = crate::input::post_message_blocked_by_uipi(target.0 as u64) {+ anyhow::bail!(msg);+ }+ let (down_msg, up_msg, mk_flag) = match button {+ "right" => (WM_RBUTTONDOWN, WM_RBUTTONUP, MK_RBUTTON),+ "middle" => (WM_MBUTTONDOWN, WM_MBUTTONUP, MK_MBUTTON),+ _ => (WM_LBUTTONDOWN, WM_LBUTTONUP, MK_LBUTTON),+ };+ let wparam = WPARAM(mk_flag as usize);+ let steps = steps.max(1);+ let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { duration_ms };+ unsafe {+ // Pre-drag MOUSEMOVE (wParam=0, no buttons down yet) then DOWN at from.+ PostMessageW(target, WM_MOUSEMOVE, WPARAM(0), make_lparam(c_from.x, c_from.y))?;+ PostMessageW(target, down_msg, wparam, make_lparam(c_from.x, c_from.y))?;+ }+ sleep(Duration::from_millis(CLICK_DELAY_MS));+ for i in 1..=steps {+ let t = i as f64 / steps as f64;+ let ix = c_from.x + ((c_to.x - c_from.x) as f64 * t).round() as i32;+ let iy = c_from.y + ((c_to.y - c_from.y) as f64 * t).round() as i32;+ unsafe { PostMessageW(target, WM_MOUSEMOVE, wparam, make_lparam(ix, iy))?; }+ if step_delay_ms > 0 {+ sleep(Duration::from_millis(step_delay_ms));+ }+ }+ unsafe {+ PostMessageW(target, up_msg, WPARAM(0), make_lparam(c_to.x, c_to.y))?;+ }+ Ok(())+}+/// Pack two 16-bit integers into a LPARAM (low word = x, high word = y).fn make_lparam(x: i32, y: i32) -> LPARAM {LPARAM((((y as u16 as u32) << 16) | (x as u16 as u32)) as isize)diff --git a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rsindex 6cff851e57..3c057fc3d1 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/overlay.rs@@ -564,6 +564,35 @@ unsafe extern "system" fn wnd_proc(.unwrap_or_default().as_millis() as u64;+ // ββ Optional overlay-FPS probe βββββββββββββββββββββββββββββββββββ+ // Set CUA_DRIVER_RS_OVERLAY_FPS_FILE=<path> to append a measured+ // render-FPS line ~once/sec. Diagnostic only; when the env var is+ // unset this is a single OnceLock read + branch (no behaviour change).+ {+ use std::sync::atomic::{AtomicU64, Ordering::Relaxed};+ static FPS_PATH: std::sync::OnceLock<Option<String>> = std::sync::OnceLock::new();+ static FPS_FRAMES: AtomicU64 = AtomicU64::new(0);+ static FPS_LAST: AtomicU64 = AtomicU64::new(0);+ if let Some(path) = FPS_PATH.get_or_init(|| std::env::var("CUA_DRIVER_RS_OVERLAY_FPS_FILE").ok()) {+ let n = FPS_FRAMES.fetch_add(1, Relaxed) + 1;+ let last = FPS_LAST.load(Relaxed);+ if last == 0 {+ FPS_LAST.store(now_ms, Relaxed);+ } else if now_ms.wrapping_sub(last) >= 1000 {+ let secs = (now_ms - last) as f64 / 1000.0;+ let fps = n as f64 / secs.max(1e-3);+ let cursors = RENDER.lock().ok()+ .and_then(|g| g.as_ref().map(|m| m.cursors.len())).unwrap_or(0);+ if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(path) {+ use std::io::Write;+ let _ = writeln!(f, "overlay fps={fps:.1} avg_dt_ms={:.1} cursors={cursors}", secs * 1000.0 / n as f64);+ }+ FPS_FRAMES.store(0, Relaxed);+ FPS_LAST.store(now_ms, Relaxed);+ }+ }+ }+// ββ Drain commands, tick all cursors, composite one pixmap βββββββ// Measure real dt from last tick β Windows timer resolution defaults// to 15ms so the hardcoded 8ms ran the animation at half speed.@@ -572,9 +601,11 @@ unsafe extern "system" fn wnd_proc(if let Some(map) = guard.as_mut() {// Drain the channel via get-or-create; track the last-touched// key so the z-order pin follows the most-recent cursor.+ let mut drained = 0u32;if let Ok(rx_guard) = CMD_RX_WIN.try_lock() {if let Some(ref rx) = *rx_guard {while let Ok(m) = rx.try_recv() {+ drained += 1;if let Some(k) = apply_msg(map, m) {map.last_active = Some(k);}@@ -597,10 +628,39 @@ unsafe extern "system" fn wnd_proc(}}- // Composite every cursor into ONE virtual-screen pixmap.- // tiny-skia fills are alpha-over, so insertion order =- // paint/z-order; idle/hidden cursors early-return inside- // paint_cursor so an idle session costs ~nothing.+ // Decide where to pin the single overlay window in z.+ //+ // The overlay is ONE full-virtual-screen layered window, so it+ // can occupy only one z-slot. It must sit ABOVE every window a+ // live cursor is actuating, but NOT above whatever sits above+ // those (the user's foreground). The right slot is therefore+ // "just above the HIGHEST-z actuating window": the overlay is+ // full-screen, so being above the topmost driven window puts it+ // above all of them (they're all at-or-below it), while still+ // below anything stacked above them. Pinning above one fixed+ // window (the old last-active behaviour) instead let any other+ // driven window stacked above it occlude its cursors β the+ // blink-out. NB: this is a RELATIVE z move (insert above a+ // specific window), which works from this non-foreground+ // thread; an absolute HWND_TOP can be refused by the+ // foreground lock and sink the overlay behind everything.+ let mut driven: Vec<u64> = Vec::new();+ for rs in map.cursors.values() {+ if !rs.core.visible || rs.core.idle_alpha < 0.004 { continue; }+ if let Some(w) = rs.core.pinned_wid {+ if !driven.contains(&w) { driven.push(w); }+ }+ }+ let pinned = unsafe { topmost_of(&driven) };++ // Composite every cursor into ONE virtual-screen pixmap and+ // blit it every frame. (An earlier "skip when idle" gate was+ // removed: starting/stopping the full-screen UpdateLayeredWindow+ // as activity comes and goes made all the resting cursors+ // flicker when any one of them clicked β very visible with many+ // cursors. A steady per-frame blit is flicker-free, and during+ // playback at least one cursor is almost always active anyway.)+ let _ = drained;let w = map.virt_w.max(1) as u32;let h = map.virt_h.max(1) as u32;let mut pm = tiny_skia::Pixmap::new(w.max(1), h.max(1))@@ -615,13 +675,6 @@ unsafe extern "system" fn wnd_proc();}- // Pin above the most-recently-touched cursor's target.- let pinned = map- .last_active- .as_ref()- .and_then(|k| map.cursors.get(k))- .and_then(|rs| rs.core.pinned_wid);-(Some(pm), arrived, pinned)} else {(None, Vec::new(), None)@@ -770,6 +823,23 @@ struct WinZOrderEnforcer {hwnd_isize: isize,}+/// Of the given window ids, return the one highest in the current z-order (the+/// first encountered walking topβbottom), or `None` if none are present. Used to+/// pick the single window the overlay should pin just above so it covers every+/// actuating window without rising above whatever sits above them.+#[cfg(target_os = "windows")]+unsafe fn topmost_of(ids: &[u64]) -> Option<u64> {+ use windows::Win32::Foundation::HWND;+ use windows::Win32::UI::WindowsAndMessaging::{GetTopWindow, GetWindow, GW_HWNDNEXT};+ if ids.is_empty() { return None; }+ let mut h = GetTopWindow(None).unwrap_or(HWND(std::ptr::null_mut()));+ while !h.0.is_null() {+ if ids.contains(&(h.0 as u64)) { return Some(h.0 as u64); }+ h = GetWindow(h, GW_HWNDNEXT).unwrap_or(HWND(std::ptr::null_mut()));+ }+ ids.first().copied()+}+impl ZOrderEnforcer for WinZOrderEnforcer {fn reassert(&self, target: Option<u64>) {#[cfg(target_os = "windows")]diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rsindex 0542127abc..b1678d28b4 100644--- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs+++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs@@ -96,6 +96,28 @@ async fn overlay_glide_to(key: &str, sx: f64, sy: f64) {crate::overlay::send_command(key.to_owned(), cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy });return;}+ // Fixed-duration glide β decouple the click from the render thread. When+ // `glide_duration_ms > 0` the path completes in exactly that wall-clock time+ // (see render_state::tick_motion), so instead of `await`-ing the render+ // thread's arrival oneshot β which couples click latency to overlay FPS and+ // degrades under many concurrent cursors β we fire the move and sleep the+ // known duration. The click then lands a deterministic time after dispatch+ // regardless of render load, while the cursor still animates visually on the+ // render thread. Speed-based glides (`== 0`, the default) keep the precise+ // arrival-await since their duration depends on distance.+ let motion = crate::overlay::current_motion(key);+ if motion.glide_duration_ms > 0.0 {+ crate::overlay::send_command(+ key.to_owned(),+ cursor_overlay::OverlayCommand::MoveTo {+ x: sx,+ y: sy,+ end_heading_radians: std::f64::consts::FRAC_PI_4,+ },+ );+ tokio::time::sleep(std::time::Duration::from_millis(motion.glide_duration_ms as u64)).await;+ return;+ }crate::overlay::animate_cursor_to(key.to_owned(), sx, sy).await;}use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef, ToolRegistry}};@@ -1825,6 +1847,19 @@ impl Tool for ClickTool {let button = args.str_or("button", "left");let count = args.u64_or("count", 1) as usize;let dispatch = DispatchMode::from_args(&args);+ // For every non-foreground click, mark the target window+ // non-activatable (WS_EX_NOACTIVATE) for the duration so a target that+ // self-activates in its UIA-Invoke / click handler (WPF+ // `UIElement.Focus()`, XAML, Tauri/WebView2) CANNOT steal the user's+ // foreground β the window still receives the click. Held for the whole+ // invoke; a no-op for dispatch:"foreground" (which wants the swap) and+ // when no window_id was given.+ let _noact = match (dispatch != DispatchMode::Foreground, hwnd_opt) {+ (true, Some(h)) => Some(crate::input::NoActivateGuard::arm(+ windows::Win32::Foundation::HWND(h as *mut _),+ )),+ _ => None,+ };// Optional `action` arg picks among the actions exposed in the// accessibility tree. Today this only changes behavior for MSAA// BUTTONDROPDOWN: `"expand"` clicks the right-edge (dropdown arrow@@ -2058,14 +2093,22 @@ impl Tool for ClickTool {}}// PostMessage fallback (legacy Win32 + non-Invokable elements).- // dispatch:"background" refuses the fallback on targets known- // to silently drop PostMessage clicks (Chromium content, GTK- // buttons). We surface a tagged error here so the outer match- // can convert to the structured background_unavailable result.+ // dispatch:"background" on targets that silently drop PostMessage+ // clicks (Chromium content, GTK buttons): route through the+ // universal coordinate-injection actuator (touch injection, no+ // foreground swap, z-order preserved) so the caller never needs+ // to know the target is Chromium/GTK and never sees a raise.+ // Only the structured error remains as a last resort (e.g. a+ // right-click, which has no clean touch mapping).if dispatch == DispatchMode::Background&& crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick){- anyhow::bail!("__CUA_BG_UNAVAILABLE_CLICK__");+ match crate::input::inject_click_screen(hwnd, cx, cy, count, &btn) {+ Ok(()) => return Ok(format!(+ "β Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)."+ )),+ Err(_) => anyhow::bail!("__CUA_BG_UNAVAILABLE_CLICK__"),+ }}crate::input::post_click_screen(hwnd, cx, cy, count, &btn)?;let action_name = match btn.as_str() {@@ -2173,18 +2216,38 @@ impl Tool for ClickTool {}}- // UIA hit-test didn't land. Decide between PostMessage / SendInput- // based on dispatch mode.+ // UIA hit-test didn't land. Decide between PostMessage / injection /+ // SendInput based on dispatch mode.//- // dispatch:"background" β refuse to swap foreground. If the target- // is known to silently drop PostMessage mouse events (Chromium- // DOM content, GTK button widgets), surface a structured- // background_unavailable error so the caller can bring_to_front- // then retry with dispatch:"foreground".+ // dispatch:"background" (the default) β never swap foreground. If the+ // target silently drops PostMessage mouse events (Chromium DOM+ // content, GTK button widgets), route through the universal+ // coordinate-injection actuator: touch injection lands in the system+ // input queue (so Chromium/Electron/WPF accept it; the OS promotes to+ // WM_*BUTTON for legacy Win32) WITHOUT SetForegroundWindow, and a+ // cloak+restore z-order guard keeps the target from visibly raising.+ // This is what lets a caller "just target the app and play actions"+ // without knowing whether it's Chromium/GTK/etc. The structured+ // background_unavailable error only survives as a last resort for+ // inputs injection can't express (e.g. right/middle clicks).if dispatch == DispatchMode::Background&& crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick){- return background_unavailable_error(hwnd, EventKind::MouseClick);+ let btn2 = btn.clone();+ let inj = tokio::task::spawn_blocking(move || {+ crate::input::inject_click_screen(hwnd, sx as i32, sy as i32, count, &btn2)+ })+ .await;+ return match inj {+ Ok(Ok(())) => {+ let click_word = match count { 2 => "double-click", 3 => "triple-click", _ => "click" };+ ToolResult::text(format!(+ "β Injected {click_word} to pid {pid} at ({sx},{sy}) (background, no foreground swap)."+ ))+ }+ Ok(Err(_)) => background_unavailable_error(hwnd, EventKind::MouseClick),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };}// dispatch:"auto" β historical heuristic: Chromium targets get@@ -2356,6 +2419,17 @@ impl Tool for TypeTextTool {is scoped per (pid, window_id). Pass the same window_id you used in \`get_window_state`.");}+ // Same no-raise guard as click: a XAML/WPF ValuePattern.SetValue handler+ // calls UIElement.Focus()βSetForegroundWindow; WS_EX_NOACTIVATE on the+ // target makes that a no-op while the value still gets set. Safe because+ // type_text never uses the SendInput foreground-swap path. No-op for+ // dispatch:"foreground" and when no window_id was given.+ let _noact = match (dispatch != DispatchMode::Foreground, hwnd_opt) {+ (true, Some(h)) => Some(crate::input::NoActivateGuard::arm(+ windows::Win32::Foundation::HWND(h as *mut _),+ )),+ _ => None,+ };let _delay_ms = args.u64_or("delay_ms", 30);let hwnd = match hwnd_opt {Some(h) => h,@@ -2419,53 +2493,60 @@ impl Tool for TypeTextTool {// uses, which we've verified works on modern Notepad / WinUI3.// Legacy Win32 stays on the PostMessage path so the no-focus-// steal property is preserved.- if crate::input::is_xaml_host_hwnd(hwnd) {- if let Some(idx) = elem_idx {- let idx = idx as usize;- let state = self.state.clone();- let text_for_uia = text.clone();- let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> {- let ptr = state.element_cache.get_element_ptr(pid, hwnd, idx)- .ok_or_else(|| anyhow::anyhow!(- "Element {idx} not in cache β call get_window_state(pid={pid}, window_id={hwnd}) first."- ))?;- use windows::Win32::UI::Accessibility::{- IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,- };- use windows::core::{Interface, BSTR};- let elem: IUIAutomationElement =- unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };- let pattern = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId)? };- std::mem::forget(elem);+ // ββ Automatic routing β the caller need not know the framework. ββ+ // 1. With an element_index, try UIA ValuePattern.SetValue first: it works+ // for WPF / WinForms / UWP / XAML and many web inputs, sets the value+ // through the accessibility channel (no keystrokes), and the `_noact`+ // guard blocks any self-foreground β so it never raises. Auto-falls-+ // back to the WM_CHAR path below if the element has no ValuePattern+ // (most legacy Win32 EDITs consume WM_CHAR fine without focus steal).+ if let Some(idx) = elem_idx {+ let idx = idx as usize;+ let state = self.state.clone();+ let text_for_uia = text.clone();+ let set_ok = tokio::task::spawn_blocking(move || -> bool {+ let Some(ptr) = state.element_cache.get_element_ptr(pid, hwnd, idx) else { return false; };+ use windows::Win32::UI::Accessibility::{+ IUIAutomationElement, IUIAutomationValuePattern, UIA_ValuePatternId,+ };+ use windows::core::{Interface, BSTR};+ let elem: IUIAutomationElement = unsafe { IUIAutomationElement::from_raw(ptr as *mut _) };+ let ok = (|| -> anyhow::Result<()> {+ let pattern = unsafe { elem.GetCurrentPattern(UIA_ValuePatternId) }?;let vp: IUIAutomationValuePattern = pattern.cast()?;- unsafe { vp.SetValue(&BSTR::from(text_for_uia.as_str()))? };+ unsafe { vp.SetValue(&BSTR::from(text_for_uia.as_str())) }?;Ok(())- }).await;- return match result {- Ok(Ok(())) => ToolResult::text(format!(- "β Wrote {text_len} char(s) on pid {raw_pid} via UIA ValuePattern \- (XAML / UWP target, element_index=[{idx}])."- )),- Ok(Err(e)) => ToolResult::error(format!("type_text (UIA path): {e}")),- Err(e) => ToolResult::error(format!("Task error: {e}")),- };- } else {- // XAML target without element_index: PostMessage will silently- // drop chars. Surface a clear error pointing the agent at the- // right workflow rather than lying with a "β Typed" message.- return ToolResult::error(format!(- "type_text on a modern XAML / UWP target (pid {raw_pid}, hwnd {hwnd}) \- requires `element_index` β its WM_CHAR pipeline ignores PostMessage \- without keyboard focus. Call `get_window_state(pid={raw_pid}, \- window_id={hwnd})` to enumerate elements, then re-call \- `type_text(pid, window_id, element_index, text)`. Or call \- `set_value(pid, window_id, element_index, value)` directly β same \- UIA backend."+ })().is_ok();+ std::mem::forget(elem);+ ok+ }).await.unwrap_or(false);+ if set_ok {+ return ToolResult::text(format!(+ "β Wrote {text_len} char(s) on pid {raw_pid} via UIA ValuePattern (element_index=[{idx}])."));}+ // ValuePattern unavailable β fall through to the WM_CHAR path.}- // Legacy Win32 path β PostMessage WM_CHAR, no focus steal.+ // 2. No element_index on a WPF target: WM_CHAR is dropped and there's no+ // element to SetValue, so deliver real keystrokes via the cloaked-+ // focus path (capability-first; the brief focus is hidden, foreground+ // restored). Supplying an element_index (path 1) is preferred and+ // never raises.+ if elem_idx.is_none() && crate::input::dispatch::is_wpf_target_window(hwnd) {+ drop(_noact);+ let text2 = text.clone();+ let r = tokio::task::spawn_blocking(move || crate::input::inject_text_cloaked(hwnd, &text2)).await;+ return match r {+ Ok(Ok(())) => ToolResult::text(format!(+ "β Typed {text_len} char(s) on pid {raw_pid} via SendInput (WPF, cloaked focus)."+ )),+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };+ }++ // 3. Legacy Win32 / GDI / Chromium-IME β PostMessage WM_CHAR, no focus steal.let result = tokio::task::spawn_blocking(move || {crate::input::post_type_text(hwnd, &text)}).await;@@ -2553,7 +2634,24 @@ impl Tool for PressKeyTool {if dispatch == DispatchMode::Background&& crate::input::dispatch::would_be_silently_dropped(hwnd, event_kind){- return background_unavailable_error(hwnd, event_kind);+ // Universal background keyboard actuator: cloaked focus + SendInput,+ // so TranslateAccelerator-based shortcuts (VCL/classic Win32) and+ // Chromium key-combos fire without a visible foreground raise. The+ // structured error only survives when focus can't be obtained+ // (foreground-lock + no UIAccess β route via the uia worker).+ let key_i = key.clone();+ let mods_i: Vec<String> = mods.clone();+ let inj = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = mods_i.iter().map(String::as_str).collect();+ crate::input::inject_key_cloaked(hwnd, &key_i, &m)+ }).await;+ return match inj {+ Ok(Ok(())) => ToolResult::text(format!(+ "β Sent {key_display} on pid {raw_pid} (background; cloaked focus if needed)."+ )),+ Ok(Err(_)) => background_unavailable_error(hwnd, event_kind),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };}// Foreground: send_key_synthesized takes the SetForegroundWindow path.if dispatch == DispatchMode::Foreground {@@ -2783,7 +2881,22 @@ impl Tool for HotkeyTool {if dispatch == DispatchMode::Background&& crate::input::dispatch::would_be_silently_dropped(hwnd, event_kind){- return background_unavailable_error(hwnd, event_kind);+ // Universal background keyboard actuator (see press_key): cloaked+ // focus + SendInput so VCL/Chromium accelerators fire without a+ // visible raise. Structured error only if focus can't be obtained.+ let key_i = key.clone();+ let mods_i: Vec<String> = mods.clone();+ let inj = tokio::task::spawn_blocking(move || {+ let m: Vec<&str> = mods_i.iter().map(String::as_str).collect();+ crate::input::inject_key_cloaked(hwnd, &key_i, &m)+ }).await;+ return match inj {+ Ok(Ok(())) => ToolResult::text(format!(+ "β Pressed {key_display} on pid {raw_pid} (background; cloaked focus if needed)."+ )),+ Ok(Err(_)) => background_unavailable_error(hwnd, event_kind),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };}// dispatch:"foreground" β explicit SendInput swap (the path that// unblocks TranslateAccelerator-style apps). Auto mode preserves the@@ -2876,6 +2989,17 @@ impl Tool for SetValueTool {None => return ToolResult::error("Missing required string field value."),};+ // No-raise guard: a WPF/XAML automation peer calls UIElement.Focus() β+ // SetForegroundWindow during ValuePattern.SetValue. WS_EX_NOACTIVATE on+ // the target makes that a no-op while the value is still set, so a+ // background SetValue can't steal the user's foreground. Held across the+ // whole write. (No-op for dispatch:"foreground".)+ let _noact = if crate::input::dispatch::DispatchMode::from_args(&args)+ != crate::input::dispatch::DispatchMode::Foreground+ {+ Some(crate::input::NoActivateGuard::arm(windows::Win32::Foundation::HWND(hwnd as *mut _)))+ } else { None };+// Glide the agent cursor onto the target element before writing its// value, so a value write gets the same visual feedback as a click β// the viewer can see *where* the agent is acting. No-op when the@@ -3546,7 +3670,7 @@ impl Tool for DragTool {})}async fn invoke(&self, args: Value) -> ToolResult {- use crate::input::dispatch::{DispatchMode, EventKind, background_unavailable_error};+ use crate::input::dispatch::{DispatchMode, EventKind};// Swift error wording 1:1.let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) {Some(p) => p,@@ -3609,11 +3733,43 @@ impl Tool for DragTool {let (sx_from, sy_from) = bitmap_to_screen(hwnd, from_x as i32, from_y as i32);let (sx_to, sy_to) = bitmap_to_screen(hwnd, to_x as i32, to_y as i32);- // dispatch:"background" β refuse if PostMessage drag would silently drop.+ // dispatch:"background" β if a PostMessage drag would silently drop+ // (Chromium/WPF/GTK canvas content reads mouse from the system input+ // queue, not the per-window queue), fall back to coordinate-routed+ // synthetic-pen drag injection instead of refusing. No foreground swap,+ // no cursor move; the target is held non-activatable + cloaked for the+ // stroke (mirrors the click pen path).if dispatch == DispatchMode::Background&& crate::input::dispatch::would_be_silently_dropped(hwnd, EventKind::MouseClick){- return background_unavailable_error(hwnd, EventKind::MouseClick);+ let target = hwnd;+ let btn = button.clone();+ pin_overlay_above(&cursor_key, hwnd);+ overlay_glide_to(&cursor_key, sx_from as f64, sy_from as f64).await;+ crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {+ x: sx_from as f64, y: sy_from as f64,+ });+ let inj = tokio::task::spawn_blocking(move || {+ crate::input::inject::inject_drag_screen(+ target, sx_from, sy_from, sx_to, sy_to, steps.max(8), &btn,+ )+ })+ .await;+ return match inj {+ Ok(Ok(())) => {+ overlay_glide_to(&cursor_key, sx_to as f64, sy_to as f64).await;+ crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse {+ x: sx_to as f64, y: sy_to as f64,+ });+ ToolResult::text(format!(+ "β Sent drag via synthetic-pen injection on pid {raw_pid} \+ from screen ({sx_from},{sy_from}) β ({sx_to},{sy_to}) \+ (dispatch:background, PostMessage would have been dropped)."+ ))+ }+ Ok(Err(e)) => ToolResult::error(e.to_string()),+ Err(e) => ToolResult::error(format!("Task error: {e}")),+ };}// dispatch:"foreground" β SendInput-based drag. Required for WPF// Slider thumbs (and any framework that polls GetKeyState during@@ -3667,10 +3823,13 @@ impl Tool for DragTool {let button_c = button.clone();let result = tokio::task::spawn_blocking(move || {- crate::input::mouse::post_drag(+ // Screen-coord, deepest-child variant: routes the gesture to the+ // child control under the start point (e.g. a WinForms Panel),+ // not the top-level frame that would ignore it.+ crate::input::mouse::post_drag_screen(hwnd,- from_x as i32, from_y as i32,- to_x as i32, to_y as i32,+ sx_from, sy_from,+ sx_to, sy_to,duration_ms, steps, &button_c,)}).await;@@ -3877,7 +4036,7 @@ impl Tool for SetAgentCursorMotionTool {Motion curve (Bezier):\n\- arc_size: perpendicular deflection as fraction of path length [0,1]. Default 0.25\n\- spring: settle damping [0.3,1.0]; 1.0=no overshoot. Default 0.72\n\- - glide_duration_ms: flight duration per move [50,5000]. Default 160\n\+ - glide_duration_ms: fixed flight duration per move [50,5000]; omit for speed-based (the default)\n\- dwell_after_click_ms: pause after click ripple [0,5000]. Default 80\n\- idle_hide_ms: auto-hide delay [0,60000]; 0=never. Default 20000".into(),input_schema: json!({@@ -3893,9 +4052,10 @@ impl Tool for SetAgentCursorMotionTool {"arc_size":{"type":"number","description":"Arc deflection as fraction of path length [0,1]. Default 0.25."},"arc_flow":{"type":"number","description":"Asymmetry bias [-1,1]. Default 0.0."},"spring":{"type":"number","description":"Settle damping [0.3,1.0]. Default 0.72."},- "glide_duration_ms":{"type":"number","minimum":50,"maximum":5000,"description":"Flight duration per move in ms. Default 160."},+ "glide_duration_ms":{"type":"number","minimum":50,"maximum":5000,"description":"Fixed flight duration per move in ms; omit for speed-based timing (the default)."},"dwell_after_click_ms":{"type":"number","minimum":0,"maximum":5000,"description":"Pause after click ripple in ms. Default 80."},- "idle_hide_ms":{"type":"number","minimum":0,"maximum":60000,"description":"Auto-hide delay in ms. 0=never. Default 20000."}+ "idle_hide_ms":{"type":"number","minimum":0,"maximum":60000,"description":"Auto-hide delay in ms. 0=never. Default 20000."},+ "turn_radius":{"type":"number","minimum":1,"maximum":1000,"description":"Minimum turning radius of the glide path in points; smaller = tighter curves. Default 80."}},"additionalProperties":false}),read_only: false, destructive: false, idempotent: true, open_world: false,@@ -3931,18 +4091,20 @@ impl Tool for SetAgentCursorMotionTool {num(args.get("dwell_after_click_ms")),num(args.get("idle_hide_ms")),None, // press_duration_ms β not in Swift tool surface+ num(args.get("turn_radius")),);crate::overlay::send_command(cursor_id.clone(), cursor_overlay::OverlayCommand::SetMotion(updated.clone()));// Match Swift text format 1:1.let summary = format!("cursor motion: startHandle={sh} endHandle={eh} arcSize={asz} arcFlow={af} \- spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih}",+ spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih} turnRadius={tr}",sh = updated.start_handle, eh = updated.end_handle,asz = updated.arc_size, af = updated.arc_flow,sp = updated.spring,gd = updated.glide_duration_ms as i64,dw = updated.dwell_after_click_ms as i64,ih = updated.idle_hide_ms as i64,+ tr = updated.turn_radius as i64,);ToolResult::text(format!("β {summary}")).with_structured(json!({"cursor_id": cursor_id,@@ -3954,6 +4116,7 @@ impl Tool for SetAgentCursorMotionTool {"glide_duration_ms": updated.glide_duration_ms,"dwell_after_click_ms": updated.dwell_after_click_ms,"idle_hide_ms": updated.idle_hide_ms,+ "turn_radius": updated.turn_radius,}))}}@@ -3989,13 +4152,14 @@ impl Tool for GetAgentCursorStateTool {// Swift text format 1:1: single-line camelCase key=value pairs.let summary = format!("cursor: enabled={enabled} startHandle={sh} endHandle={eh} arcSize={asz} \- arcFlow={af} spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih}",+ arcFlow={af} spring={sp} glideDurationMs={gd} dwellAfterClickMs={dw} idleHideMs={ih} turnRadius={tr}",sh = motion.start_handle, eh = motion.end_handle,asz = motion.arc_size, af = motion.arc_flow,sp = motion.spring,gd = motion.glide_duration_ms as i64,dw = motion.dwell_after_click_ms as i64,ih = motion.idle_hide_ms as i64,+ tr = motion.turn_radius as i64,);// Rust-only structured payload: the same fields + the multi-cursor// instance map. Cursor instances are a Rust extension Swift doesn't@@ -4012,6 +4176,7 @@ impl Tool for GetAgentCursorStateTool {"glide_duration_ms": motion.glide_duration_ms,"dwell_after_click_ms": motion.dwell_after_click_ms,"idle_hide_ms": motion.idle_hide_ms,+ "turn_radius": motion.turn_radius,"cursors": cursors,}))}