Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Desktop: expand native input, observations and background activation (#2397)

Regan Bell9e499dafbc1f
diff --git a/.github/workflows/desk-native.yml b/.github/workflows/desk-native.yml
new file mode 100644
index 000000000..46c998c3a
--- /dev/null
+++ b/.github/workflows/desk-native.yml
@@ -0,0 +1,28 @@
+name: Desktop native checks
+
+on:
+ pull_request:
+ paths:
+ - "desktop/desk/**"
+ - ".github/workflows/desk-native.yml"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ desk:
+ name: macOS desk build and tests
+ runs-on: macos-14
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd
+ with:
+ persist-credentials: false
+ - run: rustup toolchain install stable --profile minimal --component clippy,rustfmt
+ - run: cargo +stable fmt --manifest-path desktop/desk/Cargo.toml --check
+ - run: cargo +stable clippy --manifest-path desktop/desk/Cargo.toml --locked -- -D warnings
+ - run: cargo +stable test --manifest-path desktop/desk/Cargo.toml --locked
+ - run: cargo +stable build --manifest-path desktop/desk/Cargo.toml --locked --release
+ - run: swiftc desktop/desk/tests/manual/focus-witness.swift -o /tmp/desk-focus-witness
+ - run: clang -fobjc-arc -framework ApplicationServices -framework CoreGraphics -framework Foundation desktop/desk/tests/manual/focus-trace.m -o /tmp/desk-focus-trace
diff --git a/desktop/desk/Cargo.lock b/desktop/desk/Cargo.lock
index caa30cc38..0412bda36 100644
--- a/desktop/desk/Cargo.lock
+++ b/desktop/desk/Cargo.lock
@@ -2,12 +2,6 @@
# It is not intended for manual editing.
version = 4
-[[package]]
-name = "adler2"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
-
[[package]]
name = "base64"
version = "0.22.1"
@@ -16,15 +10,27 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bitflags"
-version = "1.3.2"
+version = "2.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
[[package]]
-name = "bitflags"
-version = "2.13.2"
+name = "block-buffer"
+version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "block2"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5"
+dependencies = [
+ "objc2",
+]
[[package]]
name = "cfg-if"
@@ -54,7 +60,7 @@ version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa95a34622365fa5bbf40b20b75dba8dfa8c94c734aea8ac9a5ca38af14316f1"
dependencies = [
- "bitflags 2.13.2",
+ "bitflags",
"core-foundation",
"core-graphics-types",
"foreign-types",
@@ -67,18 +73,28 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb"
dependencies = [
- "bitflags 2.13.2",
+ "bitflags",
"core-foundation",
"libc",
]
[[package]]
-name = "crc32fast"
-version = "1.5.2"
+name = "cpufeatures"
+version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
- "cfg-if",
+ "libc",
+]
+
+[[package]]
+name = "crypto-common"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
+dependencies = [
+ "generic-array",
+ "typenum",
]
[[package]]
@@ -86,25 +102,42 @@ name = "desk"
version = "0.1.0"
dependencies = [
"base64",
+ "block2",
"core-foundation",
"core-graphics",
"foreign-types",
+ "html2text",
+ "libc",
"objc2",
"objc2-app-kit",
+ "objc2-core-foundation",
+ "objc2-core-graphics",
"objc2-foundation",
- "png",
+ "objc2-screen-capture-kit",
+ "pulldown-cmark",
"serde",
"serde_json",
+ "sha2",
"signal-hook",
]
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
[[package]]
name = "dispatch2"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
dependencies = [
- "bitflags 2.13.2",
+ "bitflags",
"objc2",
]
@@ -119,24 +152,10 @@ dependencies = [
]
[[package]]
-name = "fdeflate"
-version = "0.3.7"
+name = "fastrand"
+version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
-dependencies = [
- "simd-adler32",
-]
-
-[[package]]
-name = "flate2"
-version = "1.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
-dependencies = [
- "crc32fast",
- "miniz_oxide 0.9.1",
- "zlib-rs",
-]
+checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
[[package]]
name = "foreign-types"
@@ -165,6 +184,39 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "html2text"
+version = "0.17.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7475d5c5606bb86be8d7986962ba34b233490985cfa76a20d6a23e1eb9563b1c"
+dependencies = [
+ "html5ever",
+ "tendril",
+ "thiserror",
+ "unicode-width",
+]
+
+[[package]]
+name = "html5ever"
+version = "0.40.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "456a1a377e608e555d22ddab27ac0114bc7a7b4199078108e34c2aeae6c9b130"
+dependencies = [
+ "log",
+ "markup5ever",
+ "memchr",
+]
+
[[package]]
name = "itoa"
version = "1.0.18"
@@ -178,31 +230,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
[[package]]
-name = "memchr"
-version = "2.8.3"
+name = "lock_api"
+version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
+dependencies = [
+ "scopeguard",
+]
[[package]]
-name = "miniz_oxide"
-version = "0.8.9"
+name = "log"
+version = "0.4.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
-dependencies = [
- "adler2",
- "simd-adler32",
-]
+checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
[[package]]
-name = "miniz_oxide"
-version = "0.9.1"
+name = "markup5ever"
+version = "0.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
+checksum = "0ab3dc68ac4a0f5719e560136778c1ee716e296030d75dbd4484e37e39e3a842"
dependencies = [
- "adler2",
- "simd-adler32",
+ "log",
+ "tendril",
+ "web_atoms",
]
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "new_debug_unreachable"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
+
[[package]]
name = "objc2"
version = "0.6.5"
@@ -218,9 +282,10 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c"
dependencies = [
- "bitflags 2.13.2",
+ "bitflags",
"libc",
"objc2",
+ "objc2-core-foundation",
"objc2-foundation",
]
@@ -230,11 +295,25 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
dependencies = [
- "bitflags 2.13.2",
+ "bitflags",
"dispatch2",
"objc2",
]
+[[package]]
+name = "objc2-core-graphics"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807"
+dependencies = [
+ "bitflags",
+ "dispatch2",
+ "libc",
+ "objc2",
+ "objc2-core-foundation",
+ "objc2-io-surface",
+]
+
[[package]]
name = "objc2-encode"
version = "4.1.0"
@@ -247,24 +326,104 @@ version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
dependencies = [
- "bitflags 2.13.2",
+ "bitflags",
"objc2",
"objc2-core-foundation",
]
[[package]]
-name = "png"
-version = "0.17.16"
+name = "objc2-io-surface"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d"
+dependencies = [
+ "bitflags",
+ "objc2",
+ "objc2-core-foundation",
+]
+
+[[package]]
+name = "objc2-screen-capture-kit"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74b7c5390f477482f001bc354d6571a70db7e4f8d5288e860c45521fbce11394"
+dependencies = [
+ "block2",
+ "libc",
+ "objc2",
+ "objc2-core-foundation",
+ "objc2-core-graphics",
+ "objc2-foundation",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.12.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526"
+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
- "bitflags 1.3.2",
- "crc32fast",
- "fdeflate",
- "flate2",
- "miniz_oxide 0.8.9",
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-link",
+]
+
+[[package]]
+name = "phf"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
+dependencies = [
+ "phf_shared",
+ "serde",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
]
+[[package]]
+name = "phf_generator"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
+dependencies = [
+ "fastrand",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "precomputed-hash"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+
[[package]]
name = "proc-macro2"
version = "1.0.107"
@@ -274,6 +433,24 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "pulldown-cmark"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
+dependencies = [
+ "bitflags",
+ "memchr",
+ "pulldown-cmark-escape",
+ "unicase",
+]
+
+[[package]]
+name = "pulldown-cmark-escape"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
+
[[package]]
name = "quote"
version = "1.0.47"
@@ -283,6 +460,21 @@ dependencies = [
"proc-macro2",
]
+[[package]]
+name = "redox_syscall"
+version = "0.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
[[package]]
name = "serde"
version = "1.0.229"
@@ -326,6 +518,17 @@ dependencies = [
"zmij",
]
+[[package]]
+name = "sha2"
+version = "0.10.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
[[package]]
name = "signal-hook"
version = "0.3.18"
@@ -347,10 +550,40 @@ dependencies = [
]
[[package]]
-name = "simd-adler32"
-version = "0.3.10"
+name = "siphasher"
+version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
+checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"
+
+[[package]]
+name = "smallvec"
+version = "1.16.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b"
+
+[[package]]
+name = "string_cache"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6"
+dependencies = [
+ "new_debug_unreachable",
+ "parking_lot",
+ "phf_shared",
+ "precomputed-hash",
+]
+
+[[package]]
+name = "string_cache_codegen"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "928dcdf75e47626b3617a976ec205d9f057584c371c1f23b782129268d0e6edc"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2",
+ "quote",
+]
[[package]]
name = "syn"
@@ -363,12 +596,77 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "tendril"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08"
+dependencies = [
+ "new_debug_unreachable",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "2.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "typenum"
+version = "1.20.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
+
+[[package]]
+name = "unicase"
+version = "2.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f"
+
[[package]]
name = "unicode-ident"
version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
+[[package]]
+name = "unicode-width"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
+
+[[package]]
+name = "version_check"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+
+[[package]]
+name = "web_atoms"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7572660c8890448ba236b7376f27e389c6a7e1c70195622faced601f855c0ada"
+dependencies = [
+ "phf",
+ "phf_codegen",
+ "string_cache",
+ "string_cache_codegen",
+]
+
[[package]]
name = "windows-link"
version = "0.2.1"
@@ -384,12 +682,6 @@ dependencies = [
"windows-link",
]
-[[package]]
-name = "zlib-rs"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
-
[[package]]
name = "zmij"
version = "1.0.23"
diff --git a/desktop/desk/Cargo.toml b/desktop/desk/Cargo.toml
index 22d451c4b..7c0d20a5f 100644
--- a/desktop/desk/Cargo.toml
+++ b/desktop/desk/Cargo.toml
@@ -10,15 +10,22 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
base64 = "0.22"
signal-hook = "0.3"
+sha2 = "0.10"
+pulldown-cmark = { version = "0.13", default-features = false, features = ["html"] }
+html2text = { version = "0.17", default-features = false }
[target.'cfg(target_os = "macos")'.dependencies]
+libc = "0.2"
core-foundation = "0.10"
core-graphics = { version = "0.24", features = ["highsierra", "elcapitan"] }
foreign-types = "0.5"
-png = "0.17"
objc2 = "0.6"
-objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSString", "NSArray", "NSEnumerator"] }
-objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSRunningApplication", "NSWorkspace"] }
+objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSString", "NSDate", "NSDateFormatter", "NSFormatter", "NSCalendar", "NSBundle", "NSArray", "NSEnumerator", "NSData", "NSError"] }
+block2 = "0.6"
+objc2-core-foundation = { version = "0.3", default-features = false, features = ["std", "CFCGTypes"] }
+objc2-core-graphics = { version = "0.3", default-features = false, features = ["std", "CGImage", "CGEvent", "CGEventTypes", "CGWindow", "CGSession", "libc"] }
+objc2-screen-capture-kit = { version = "0.3", default-features = false, features = ["std", "libc", "block2", "objc2-core-foundation", "objc2-core-graphics", "SCShareableContent", "SCStream", "SCScreenshotManager"] }
+objc2-app-kit = { version = "0.3", default-features = false, features = ["std", "libc", "NSRunningApplication", "NSWorkspace", "NSPasteboard", "NSPasteboardItem", "NSEvent", "NSGraphicsContext", "NSFont", "NSFontDescriptor", "objc2-core-foundation"] }
[profile.release]
opt-level = "z"
diff --git a/desktop/desk/PROTOCOL.md b/desktop/desk/PROTOCOL.md
index ca41bc39b..8a4fbb960 100644
--- a/desktop/desk/PROTOCOL.md
+++ b/desktop/desk/PROTOCOL.md
@@ -16,12 +16,13 @@ Command = permissions { prompt?: bool = false }
Action = click { target: Target, button?: left|right|middle = left, count?: u8 = 1,
modifiers?: [command|shift|option|control], hold_ms?: u32 = 0 }
| type_text { text: string }
+ | paste { text: string, format?: text|markdown|html = text }
| press_key { key: string, hold_ms?: u32 = 0 } // a chord such as "cmd+shift+t"
| scroll { target?: Target, direction: up|down|left|right, amount?: Amount = { "pages": 1 } }
| select_text { index: u32, text: string, prefix?: string, suffix?: string,
selection?: text|cursor_before|cursor_after = text }
| drag { from: Point, to: Point }
- | set_value { index: u32, value: string }
+ | set_value { index: u32, value: string, submit?: bool }
| perform { index: u32, action: string } // any AX action the element lists, e.g. "ShowMenu", "Increment"
| raise // bring the app's window to the front (steals focus)
| wait { ms: u32 } // capped at 10000
@@ -31,7 +32,7 @@ Amount = { "pages": f64 } | { "pixels": u32 }
Point = { x: f64, y: f64 }
```
-`app` matches a running app's name or bundle id, case-insensitive. An installed app that is not running is opened in the background (`open -g`) first. A `press_key` chord is modifiers (`cmd`, `shift`, `opt`/`alt`, `ctrl`) joined by `+` and ending in a key, which is a letter, digit, punctuation, `return`, `tab`, `space`, `delete`, `escape`, `forwarddelete`, `home`, `end`, `pageup`, `pagedown`, arrows (`left` …), or `f1`–`f12`.
+`app` matches a running app's name or bundle id, case-insensitive, or its full application path (resolved through symlinks). Ambiguous matches return the candidate PIDs, bundle IDs and paths without sending input. An installed app that is not running is opened in the background (`open -g`) first. A `press_key` chord is modifiers (`cmd`, `shift`, `opt`/`alt`, `ctrl`) joined by `+` and ending in a key, which is a letter, digit, punctuation, `return`, `tab`, `space`, `delete`, `escape`, `forwarddelete`, `home`, `end`, `pageup`, `pagedown`, arrows (`left` …), or `f1`–`f20`. Keypad names such as `KP_Enter`, `KP_0` and `KP_Add`, X11 punctuation names such as `exclam`, and side-specific modifier names such as `Control_R` are also accepted.
Example:
@@ -53,41 +54,80 @@ Reply = permissions { accessibility: bool, screen_recording: bool, binary: st
| error { ...Failure }
AppInfo { name, bundle_id?, pid: i32, active: bool }
-Observation { app, bundle_id?, pid, window: string, frame: Rect, elements: u32, truncated: bool,
+Observation { app, bundle_id?, pid, window: string, window_id: u32, frame: Rect, elements: u32, truncated: bool,
diffed: bool, tree: string, removed: [u32], screenshot: Screenshot? }
Rect { x, y, w, h } // global screen points, top-left origin
-Screenshot { png_base64, width, height, pixels_per_point }
+Screenshot { png_base64, width, height, pixels_per_point, frame: Rect } // frame = global screen points the image covers
Failure { code: ErrorCode, message: string, action: u32? } // action = index of the failed action in a batch
ErrorCode = bad_request | unsupported_platform | permission_missing | app_not_found
- | no_window | bad_index | unknown_key | action_failed
+ | no_window | bad_index | unknown_key | action_failed | paste_conflict
```
## Observation semantics
-- The tree covers the app's focused window (else main, else first). One line per kept element:
- `[id] Role "title" (description) = "value" [focused disabled selected settable] <Actions> @x,y wxh`.
- Unlabelled containers are folded away; indentation shows nesting. Capped at 800 elements (`truncated: true`).
-- Frames and `point` targets are **window-relative points** (origin = window top-left).
- The screenshot is captured at nominal resolution, so one PNG pixel is about one point; `pixels_per_point` says what it really was.
- Divide PNG pixels by `pixels_per_point` before sending a `point`.
-- Observe before acting; absent snapshots are rejected. Ids are stable: an element keeps its id across observations of the same window for as long as it is the same AX object. A different window starts ids again. `index` in actions means this id, from the latest observation of that app.
-- With `diff` (the default) and an earlier observation of the same window, `tree` holds only new or changed lines, `removed` lists ids that disappeared and `diffed` is true. Otherwise `tree` is the whole tree.
-- After the actions, `act` waits 1 s, then up to 5 s more while the window shows a busy or progress indicator, whether or not it observes.
+- The tree covers the visible part of the app's focused window (else main, else the last entry in its accessibility window list). A minimized selected window is refused rather than replaced by another document. One line per kept element:
+ `[id] Role:Subrole "name" (description) = "value" placeholder "…" url "…" [focused disabled selected checked|unchecked|mixed expanded|collapsed settable] <Actions>`.
+ The name is `AXTitle`, else the text of the element's `AXTitleUIElement` label. Checkbox, radio and switch values are
+ shown as `checked`/`unchecked`/`mixed`, not `= "1"`. Text roles keep values up to 1,000 characters, others 120.
+ Unlabelled containers are folded away; indentation shows nesting. A heading reads `[id] ## text` (Markdown, by its
+ level). A heading or link with no title whose children are only plain text takes that text as its name, and the
+ text lines are not repeated under it. Items of a web list (`AXList` with subrole `AXContentList`) that are plain
+ groups read `[id] - text`, or `[id] -` with their content nested under it.
+- Neighbouring plain static texts (no actions, not focused or selected) merge into one `StaticText` line carrying the
+ first one's id; a fragment that starts on a lower line than the previous one is joined with `⏎`, others verbatim.
+- Open menus (context menus and menu-bar menus) and the app's other sheets, popovers, drawers and dialog or floating
+ windows come first, under `Open menus and other windows of this app:`, followed by `Window "title":` and the
+ window's tree. Without any, the tree starts directly.
+- After the tree, `Focus: [id]` names the app's focused element (or describes it when it is not in the tree), and
+ `Selected text: "…"` quotes its selection when there is one.
+- Children outside the window, or outside an enclosing `AXScrollArea` or `AXWebArea`, are not walked. Each container
+ that had some gets one note line, `… N more offscreen; scroll to reveal`. Zero-size frames count as offscreen unless
+ strictly inside the visible area (Chromium pins scrolled-out elements to the viewport edge with zero height).
+- The tree is capped at about 60,000 characters of lines and 6,000 AX elements read (`truncated: true`). The characters
+ are shared fairly: siblings that fit are shown whole, and the rest split what is left evenly, so one long list cannot
+ starve the containers after it. Every container cut short gets `… N more not shown; the observation size limit was
+reached`.
+- A web area, or a childless group or scroll area, covering at least 40% of the window with nothing readable inside gets
+ `… nothing readable inside; the app may draw this area itself (for example a canvas). Use the screenshot and point targets`.
+- Each element's attributes are read in one batched call; actions and settability only for elements that are walked.
+ While desk is subscribed to an app's accessibility notifications, it reuses each element's actions and settability
+ from earlier observations until the app posts any notification, which clears them all; without a subscription it
+ reads them fresh every time.
+ On the first observation of an app desk sets `AXManualAccessibility` and waits up to 1 s for an empty web area to
+ fill. Only if that attribute is refused and a web area is still empty does it set `AXEnhancedUserInterface` and wait
+ again. Whichever it turned on is set back to off when desk exits; attributes that were already on are left alone.
+- `point` targets are **window-relative points** (origin = window top-left).
+ On macOS 14+, the screenshot covers its own `frame`: the window plus the app's menus, popovers and sheets in front of it, so it can reach past the window. On macOS 12.3–13 it covers only the main window.
+ It is captured at nominal resolution, so one PNG pixel is about one point; `pixels_per_point` says what it really was.
+ To send a `point`, divide PNG pixels by `pixels_per_point`, then add the screenshot `frame` origin minus the observation `frame` origin. `window_id` identifies the observed native window. The JavaScript API requires a screenshot before point actions and invalidates its image mapping when a later observation reports a different process, window ID or frame. This check is shared across JavaScript App objects by the observed bundle ID (or app name when no bundle ID exists), so using a different selector cannot reuse stale coordinates. Multiple instances sharing that identity may require a new screenshot after observing another instance.
+- Observe before acting; absent snapshots are rejected. Ids are stable: an element keeps its id across observations of the same window for as long as it is the same AX object, including while it is scrolled out of view for up to 30 observations. When the app replaces an object (a re-render), the new one inherits the id if exactly one old object and exactly one new object have the same role, subrole, identifier, name and ancestry, the old object is gone, and neither sits inside a list, table, outline, row or cell. An inherited id's line is always marked `~` in a diff. Acting on an id whose object is gone looks for that single replacement first and fails with `bad_index` if there is none. A different window starts ids again. `index` in actions means this id, from the latest observation of that app.
+- With `diff` (the default) and an earlier observation of the same window, `tree` holds only new lines, prefixed `+ `, and changed ones, prefixed `~ `; `removed` lists ids that disappeared and `diffed` is true. When more than 60% of the lines are new or changed, a selection/state/note line disappears, or no earlier observation is available, `tree` is the whole tree, unprefixed, and `diffed` is false.
+- `act` returns as soon as its actions are sent. The next observation of that app (`observe`, or `act` with `observe`) first waits for that app's accessibility notifications to go quiet (other apps' notifications do not count): at least 200 ms after the last action, then until 300 ms pass with no notification, at most 2 s after the action. If desk could not subscribe to the app's notifications it waits until 1 s has passed instead. Then it waits up to 5 s more while the window shows a busy or progress indicator. An observation with no action since the previous one does not wait. A batch of only `wait` actions does not count as an action.
- A batch stops at the first failed action. `acted.completed` counts what ran; the observation or `observation_failure` comes back without losing the completion count.
## Pacing
-With `pace: true`, before each `click`, `scroll` and `drag`, desk writes `pointer` with the request's id, the action's index and the global screen point it is about to use (element centre, window centre for an untargeted scroll, `from` for a drag). It then reads exactly one line, which must be `{"id": <same id>, "cmd": {"type": "arrived"}}`, and only then performs the action. Anything else fails that action with `bad_request`. The QM desktop app uses this to fly its cursor overlay to the point first. Send nothing else while a paced act is running.
+With `pace: true`, before each `click`, `scroll` and `drag`, desk writes `pointer` with the request's id, the action's index and the global screen point it is about to use (the element's clickable point, window centre for an untargeted scroll, `from` for a drag). It then reads exactly one line, which must be `{"id": <same id>, "cmd": {"type": "arrived"}}`, and only then performs the action. Anything else fails that action with `bad_request`. The QM desktop app uses this to fly its cursor overlay to the point first. Send nothing else while a paced act is running.
## How actions are delivered
-- `click` on an index, left button, single: `AXPress` if the element offers it (no pointer, no focus change); otherwise a synthetic click at the element's center.
-- Other clicks, `drag`, `scroll`, `type_text`, `press_key`: CGEvents posted to the app's pid (`CGEventPostToPid`). The real pointer does not move and the app is not brought forward. Some apps ignore events while in the background; then send `raise` first.
+- `click` on an index, left button, single: menu items always use `AXPress`. Elements inside a web area, rows, cells and list items, and elements without `AXPress` get a synthetic click, because `AXPress` there fires no pointer events, focus change or selection. Other elements use `AXPress` (no pointer, no focus change).
+- The point of a synthetic click on an index is found by hit-testing the centre and four inner points of the element's visible part and taking the first where the app reports the element or one of its descendants. If none of the five probes identifies the target or a descendant, the action fails with `action_failed`; failed, ancestor-only and unrelated hits are insufficient. Transparent containers without a target/descendant hit require an explicit screenshot point. This requirement does not apply to the AXPress path.
+- A synthetic-click index target that is offscreen (outside the window or an enclosing scroll or web area) is first sent `AXScrollToVisible`; if it is still offscreen 250 ms later, the action fails with `action_failed`.
+- Other clicks, `drag`, `scroll`, `type_text`, `press_key`, and the Cmd+V of `paste`: CGEvents posted to the app's pid (`CGEventPostToPid`). The real pointer does not move. Desk retains intervention protection for direct AX actions and sends target-only synthetic activation only when a dispatch needs synthetic input. Direct AX actions do not activate or deactivate the app. Synthetic activation does not deliberately change the frontmost app. A batch that sends synthetic input retains its normal focus cleanup. Some apps may still ignore these events.
+- Background input protects the observed key-focus lease: a process-notification tap suppresses the target's focus-taken notification, releases the matching lease by its theft ID, and suppresses its paired return notification. External key events misdirected to that target during the lease are forwarded by its process-local keyboard tap to the prior owner; agent events and unrelated input pass unchanged. A separate global listen-only tap detects human intervention. Target registration waits for its keyboard tap to be installed before input; all taps are disabled and removed at scope teardown. Missing private API support, tap failure, nested leases, or a genuine foreground change stop the batch. This is not a timer that reactivates the previous app, and it does not prevent arbitrary application window reordering.
+- Pointer events at a point where the app's own hit test finds an Open or Save panel drawn by `com.apple.appkit.xpc.openAndSavePanelService` inside the app's window go to that panel's process and window. Keystrokes, `paste` and `set_value` submission go to that process when the app's focused element belongs to it.
+- While the app has a menu open (a context or pop-up menu, or a menu-bar menu), only `click` on a menu item by index works (with `AXPress`); other clicks, non-menu `perform` `AXPress` actions, and every action that would send pointer events fail with `action_failed`. Press Escape to close the menu. Nested popup detection checks current children of popup buttons retained from the latest observation, with at most 128 candidate and child checks; unreadable state refuses pointer input, and exceeding this bound remains unsupported even after another observation. Newly created or previously unobserved popup buttons require a fresh observation.
+- Between actions of one `act`, except next to a `wait`, desk waits until the app has sent no accessibility notification (focus, value, selection, menu, window, layout, creation) for 120 ms, at least 60 ms and at most 800 ms, so a later action sees the UI an earlier one opened.
+- `type_text` sends each character with the key code and Shift/Option flags that type it on the user's current keyboard layout, plus the character itself. `\n`, `\r\n` and `\r` press Return and `\t` presses Tab. Characters the layout cannot type (emoji, other scripts, dead-key accents) are sent as text on key code 0.
+- `paste` puts the text on the general pasteboard (plain text, plus HTML for `markdown` and `html`; markdown is converted to HTML), marked `org.nspasteboard.TransientType` so clipboard managers skip it, and presses Cmd+V. The data is written eagerly before the shortcut, so the receiving app can read it without waiting for a callback from desk. A clipboard read is not taken as proof of delivery. desk restores the previous clipboard only after the app's focused field shows its previous text with the pasted text (the plain text, or for `markdown` and `html` also the text of the HTML) inserted exactly at the selection; only line endings and non-breaking spaces are normalised. Otherwise (the field never shows the text within 3 s, the focused field's text cannot be read, a failure while sending Cmd+V, or cancellation or interruption after it) the action fails with `action_failed`, desk's text stays on the clipboard and the previous clipboard is not restored, so a late paste cannot insert the user's clipboard. If the clipboard changes for another reason meanwhile, desk keeps that new content and fails with `paste_conflict`, because the app may have pasted it instead. If the clipboard holds a concealed item (`org.nspasteboard.ConcealedType`, which password managers use), `paste` fails without touching the clipboard.
- `click` holds `modifiers` and keeps the button down for `hold_ms`. `press_key` holds the key for `hold_ms`.
-- `scroll` sends pixel wheel events in 120 px steps; a page is 85% of the target element's height (or width), else the window's.
-- `select_text` finds `text` in the element's `AXValue` (the occurrence preceded by `prefix` and followed by `suffix`), focuses the element and sets `AXSelectedTextRange` to the text or a caret before or after it.
-- `set_value` writes `AXValue`. `perform` runs any listed AX action.
+- `scroll` by pages first scrolls the enclosing scroll area (of the target element, or at the target point or window centre) without pointer events: its `AXScroll…ByPage` action for whole pages, else by moving its scroll bar by 85% of the visible part per page. Otherwise, and for `pixels`, it sends pixel wheel events in 120 px steps; a page is 85% of the target element's height (or width), else the window's.
+- `select_text` accepts nonempty native text or an exact excerpt of an observed Markdown body, excluding structural tree indentation. Matches in `AXValue` and observed Markdown are combined after filtering by adjacent `prefix` and `suffix` within each representation, then deduplicated by native UTF-16 range; exactly one range is required. Renderer provenance maps escaped text and link labels without interpreting literal Markdown, and generated syntax or link destinations alone cannot select text. A SHA-256 fingerprint of the full native value invalidates its observed mapping when the value changes and requires another observation. Mapping allocations have a 1 MiB aggregate observation budget; controls exceeding it use the plain-text observation fallback. It checks that `AXSelectedTextRange` is writable before focusing the element, verifies focus and unchanged text, then selects the match or places a caret before or after it. Unsupported selection, ambiguous matches and failed focus return an error without writing a selection.
+- `set_value` fails unless `AXValue` is settable. It writes the value as the type the element holds: a number (`true`/`false` count as 1/0) or boolean, a date, or otherwise a string. Dates use the system’s short date/time format, with seconds included when the control exposes them (full month name and year for month/year controls), changing only the components exposed by the control. Invalid dates and changes that would normalize a preserved component are refused before mutation. Text fields are focused and their focus is verified before writing; interruption is checked immediately before the value write. If a text field still shows its old value afterwards the action fails. Only with `submit: true` desk then sends `AXConfirm`, or Return when the element lacks it and still has keyboard focus. `perform` validates its name against the element’s current AX actions before running it. The primary `Press` action is omitted from observation text because `click` already invokes it; secondary actions remain listed.
## App access
All installed apps are available subject to macOS permissions. Observation and control require Accessibility; screenshots also require Screen Recording.
+
+On macOS 14 and later, screenshots use display-relative crops of the selected window and associated transients. Windows outside the current Space are refused; portions outside all displays remain transparent.
diff --git a/desktop/desk/README.md b/desktop/desk/README.md
index 03cafcada..b891b8106 100644
--- a/desktop/desk/README.md
+++ b/desktop/desk/README.md
@@ -19,13 +19,15 @@ open -a Calculator
Run alone, macOS grants permissions to the terminal that launches desk, so switch your terminal on under System Settings > Privacy & Security > Accessibility and Screen Recording, then restart it.
-All running apps can be observed and controlled, subject to macOS permissions.
+App listing shows regular Mac apps. Explicit names, bundle IDs, and full application paths can also target accessory apps with an accessible, visible window. Background-only processes are excluded; observation and control remain subject to macOS permissions and app support.
## Platform considerations
-- **Events posted to a background app.** `CGEventPostToPid` clicks and keys might be ignored, or land in the wrong window, when the app isn't frontmost. If nothing happens, add `{"type":"raise"}` first; that does take focus.
-- **Screenshots.** They use `CGWindowListCreateImage`, deprecated in macOS 14 in favour of ScreenCaptureKit. On macOS 15+ it may return nothing or prompt monthly. `pixels_per_point` also needs checking on a Retina screen.
+- **Background input is experimental.** Desk sends target-only synthetic activation and input without raising the app, but some apps may ignore it. `{"type":"raise"}` is an explicit foreground action. Live Mac verification is still required.
+- **Screenshots.** On macOS 14+ they use ScreenCaptureKit's `SCScreenshotManager`. The app's own menus, popovers and sheets in front of the target window are composited in, and the image grows to include any part outside the window, across displays if needed; `frame` says which screen area it covers. macOS 12.3–13 uses `CGWindowListCreateImage` for the main window only; menu/panel composition requires macOS 14+. desk links ScreenCaptureKit, so it needs macOS 12.3 or later. `pixels_per_point` still needs checking on a Retina screen.
- **`_AXUIElementGetWindow`** (private, widely used) maps the AX window to a window id for capture and event targeting.
-- **`AXManualAccessibility`** is set on each app so Electron and Chrome apps expose their trees.
-- **Speed.** Each element costs about 7 AX round trips, so big windows may take seconds. The 800-element cap bounds this.
+- **Clipboard.** `paste` replaces the general pasteboard, presses Cmd+V and restores the previous contents only once the focused field visibly contains the pasted text. When it cannot confirm that (canvas or terminal views without readable text, password fields, slow apps), it leaves the pasted text on the clipboard and says so, rather than risk the app pasting the user's clipboard. Clipboard data is available before Cmd+V is sent, so the receiving app does not need a callback from desk to read it.
+- **Keyboard layout.** `type_text` reads the current layout with `TISCopyCurrentKeyboardLayoutInputSource` and `UCKeyTranslate` on desk's main thread (TIS calls are main-thread only on recent macOS).
+- **Accessibility enablement.** On first observation, desk requests `AXManualAccessibility` and uses `AXEnhancedUserInterface` only if an empty web area still needs it. Prior settings are restored on normal exit.
+- **Speed.** Element attributes are read in batches. Actions and settability are cached until that app posts an accessibility notification; frames, values and children are always read fresh. Offscreen pruning and a 6,000-element read cap bound each walk.
- **Universal binary:** `cargo build --release --target aarch64-apple-darwin --target x86_64-apple-darwin`, then `lipo`.
diff --git a/desktop/desk/src/capture.rs b/desktop/desk/src/capture.rs
new file mode 100644
index 000000000..a3f36f62b
--- /dev/null
+++ b/desktop/desk/src/capture.rs
@@ -0,0 +1,365 @@
+use crate::cancellation;
+use crate::protocol::{Rect, Screenshot};
+use crate::tree::intersect as intersection;
+use base64::Engine;
+use block2::RcBlock;
+use core_foundation::array::CFArray;
+use core_foundation::base::{CFRetain, CFType, CFTypeRef, TCFType};
+use core_foundation::dictionary::CFDictionary;
+use core_foundation::number::CFNumber;
+use core_foundation::runloop::{kCFRunLoopDefaultMode, CFRunLoop};
+use core_foundation::string::{CFString, CFStringRef};
+use core_graphics::base::kCGImageAlphaPremultipliedLast;
+use core_graphics::color_space::{kCGColorSpaceSRGB, CGColorSpace};
+use core_graphics::context::CGContext;
+use core_graphics::image::CGImage;
+use core_graphics::window::{
+ copy_window_info, create_image, kCGWindowImageBoundsIgnoreFraming, kCGWindowImageNominalResolution, kCGWindowListOptionIncludingWindow,
+ kCGWindowListOptionOnScreenAboveWindow, kCGWindowNumber,
+};
+use foreign_types::ForeignType;
+use objc2::rc::Retained;
+use objc2::runtime::AnyClass;
+use objc2::AnyThread;
+use objc2_foundation::{NSArray, NSError, NSMutableData};
+use objc2_screen_capture_kit::{SCContentFilter, SCDisplay, SCScreenshotManager, SCShareableContent, SCStreamConfiguration, SCWindow};
+use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender};
+use std::time::{Duration, Instant};
+
+#[link(name = "ImageIO", kind = "framework")]
+extern "C" {
+ fn CGImageDestinationCreateWithData(data: CFTypeRef, kind: CFStringRef, count: usize, options: CFTypeRef) -> CFTypeRef;
+ fn CGImageDestinationAddImage(destination: CFTypeRef, image: CFTypeRef, properties: CFTypeRef);
+ fn CGImageDestinationFinalize(destination: CFTypeRef) -> bool;
+}
+
+const TIMEOUT: Duration = Duration::from_secs(5);
+const MAX_EDGE: f64 = 32_768.0;
+
+struct Piece {
+ image: CGImage,
+ at: Rect,
+}
+
+unsafe impl Send for Piece {}
+
+enum Part {
+ Plan { area: Rect, pieces: usize },
+ Piece(Piece),
+}
+
+type Parts = Sender<Result<Part, String>>;
+
+pub fn window(id: u32, panels: &[u32], frame: Rect) -> Result<Screenshot, String> {
+ size(frame)?;
+ if AnyClass::get(c"SCScreenshotManager").is_none() {
+ return legacy(id, frame);
+ }
+ let above = above(id);
+ let panels = panels.to_vec();
+ let (tx, rx) = channel();
+ let block = RcBlock::new(move |content: *mut SCShareableContent, error: *mut NSError| {
+ let planned = unsafe { content.as_ref() }.ok_or_else(|| reason(error)).and_then(|c| plan(c, id, &above, &panels, frame, &tx));
+ if let Err(e) = planned {
+ let _ = tx.send(Err(e));
+ }
+ });
+ unsafe { SCShareableContent::getShareableContentExcludingDesktopWindows_onScreenWindowsOnly_completionHandler(true, false, &block) };
+ drop(block);
+ let (area, pieces) = collect(rx)?;
+ let image = compose(area, &pieces)?;
+ encode(&image, area)
+}
+
+fn above(id: u32) -> Vec<u32> {
+ let number = |d: &CFDictionary<CFString, CFType>, key| d.find(key).and_then(|v| v.downcast::<CFNumber>()).and_then(|n| n.to_i64());
+ let infos = copy_window_info(kCGWindowListOptionOnScreenAboveWindow | kCGWindowListOptionIncludingWindow, id);
+ infos
+ .iter()
+ .flat_map(CFArray::iter)
+ .filter_map(|item| {
+ let d = unsafe { CFDictionary::<CFString, CFType>::wrap_under_get_rule(*item as _) };
+ u32::try_from(number(&d, unsafe { CFString::wrap_under_get_rule(kCGWindowNumber) })?).ok()
+ })
+ .collect()
+}
+
+fn plan(content: &SCShareableContent, id: u32, above: &[u32], panels: &[u32], frame: Rect, tx: &Parts) -> Result<(), String> {
+ let windows = unsafe { content.windows() };
+ let target = windows.iter().find(|w| unsafe { w.windowID() } == id).ok_or("The window is not available to capture.")?;
+ if !unsafe { target.isOnScreen() } {
+ return Err("The window is not visible on the current Space; show it before capturing.".into());
+ }
+ let candidates: Vec<(u32, Rect)> = windows
+ .iter()
+ .filter_map(|w| {
+ let wid = unsafe { w.windowID() };
+ (wid != id && unsafe { w.isOnScreen() } && above.contains(&wid)).then(|| (wid, rect(unsafe { w.frame() })))
+ })
+ .collect();
+ let (ids, area) = associated(frame, &candidates, panels);
+ size(area)?;
+ let shown: Vec<Retained<SCWindow>> =
+ windows.iter().filter(|w| unsafe { w.windowID() } == id || ids.contains(&unsafe { w.windowID() })).collect();
+ let shown = NSArray::from_retained_slice(&shown);
+ let parts: Vec<(Retained<SCDisplay>, Rect)> =
+ unsafe { content.displays() }.iter().filter_map(|d| intersection(rect(unsafe { d.frame() }), area).map(|r| (d, r))).collect();
+ if parts.is_empty() {
+ return Err("The window is not on any display.".into());
+ }
+ tx.send(Ok(Part::Plan { area, pieces: parts.len() })).map_err(|e| e.to_string())?;
+ for (display, at) in parts {
+ let filter = unsafe { SCContentFilter::initWithDisplay_includingWindows(SCContentFilter::alloc(), &display, &shown) };
+ let origin = rect(unsafe { display.frame() });
+ grab(&filter, at, Rect { x: at.x - origin.x, y: at.y - origin.y, w: at.w, h: at.h }, tx.clone());
+ }
+ Ok(())
+}
+
+fn grab(filter: &SCContentFilter, at: Rect, source: Rect, tx: Parts) {
+ let config = unsafe { SCStreamConfiguration::new() };
+ unsafe {
+ config.setWidth(at.w.round().max(1.0) as usize);
+ config.setHeight(at.h.round().max(1.0) as usize);
+ config.setShowsCursor(false);
+ config.setIgnoreShadowsDisplay(true);
+ config.setIgnoreShadowsSingleWindow(true);
+ config.setSourceRect(cg(source));
+ }
+ let done = RcBlock::new(move |image: *mut objc2_core_graphics::CGImage, error: *mut NSError| {
+ let piece = match unsafe { image.as_ref() } {
+ Some(i) => Ok(Part::Piece(Piece {
+ image: unsafe { CGImage::from_ptr(CFRetain((i as *const objc2_core_graphics::CGImage).cast()) as _) },
+ at,
+ })),
+ None => Err(reason(error)),
+ };
+ let _ = tx.send(piece);
+ });
+ unsafe { SCScreenshotManager::captureImageWithFilter_configuration_completionHandler(filter, &config, Some(&done)) };
+}
+
+fn collect(rx: Receiver<Result<Part, String>>) -> Result<(Rect, Vec<Piece>), String> {
+ let start = Instant::now();
+ let (mut plan, mut pieces) = (None, vec![]);
+ loop {
+ if let Some((area, n)) = plan {
+ if pieces.len() == n {
+ return Ok((area, pieces));
+ }
+ }
+ if cancellation::cancelled() {
+ return Err("Computer action cancelled".into());
+ }
+ match rx.recv_timeout(Duration::from_millis(20)) {
+ Ok(Ok(Part::Plan { area, pieces: n })) => plan = Some((area, n)),
+ Ok(Ok(Part::Piece(p))) => pieces.push(p),
+ Ok(Err(e)) => return Err(e),
+ Err(RecvTimeoutError::Disconnected) => return Err("Screen capture ended without an image.".into()),
+ Err(RecvTimeoutError::Timeout) if start.elapsed() >= TIMEOUT => return Err("Screen capture timed out.".into()),
+ Err(RecvTimeoutError::Timeout) => {
+ CFRunLoop::run_in_mode(unsafe { kCFRunLoopDefaultMode }, Duration::ZERO, true);
+ }
+ }
+ }
+}
+
+fn size(r: Rect) -> Result<(usize, usize), String> {
+ let (w, h) = (r.w.round(), r.h.round());
+ let usable = [r.x, r.y, r.w, r.h].iter().all(|v| v.is_finite())
+ && (1.0..=MAX_EDGE).contains(&w)
+ && (1.0..=MAX_EDGE).contains(&h)
+ && w * h <= 64_000_000.0;
+ if usable {
+ Ok((w as usize, h as usize))
+ } else {
+ Err(format!("The window frame {}x{} is not a size that can be captured.", r.w, r.h))
+ }
+}
+
+fn compose(area: Rect, pieces: &[Piece]) -> Result<CGImage, String> {
+ let (w, h) = size(area)?;
+ let failed = || "Could not compose the screenshot.".to_string();
+ let space = CGColorSpace::create_with_name(unsafe { kCGColorSpaceSRGB }).ok_or_else(failed)?;
+ let context = CGContext::create_bitmap_context(None, w, h, 8, 0, &space, kCGImageAlphaPremultipliedLast);
+ let h = h as f64;
+ context.clear_rect(core_graphics::geometry::CGRect::new(
+ &core_graphics::geometry::CGPoint::new(0.0, 0.0),
+ &core_graphics::geometry::CGSize::new(w as f64, h),
+ ));
+ for p in pieces {
+ let at = Rect { x: p.at.x - area.x, y: h - (p.at.y - area.y) - p.at.h, w: p.at.w, h: p.at.h };
+ context.draw_image(
+ core_graphics::geometry::CGRect::new(
+ &core_graphics::geometry::CGPoint::new(at.x, at.y),
+ &core_graphics::geometry::CGSize::new(at.w, at.h),
+ ),
+ &p.image,
+ );
+ }
+ context.create_image().ok_or_else(failed)
+}
+
+fn reason(error: *mut NSError) -> String {
+ unsafe { error.as_ref() }.map_or_else(|| "Screen capture returned nothing.".into(), |e| e.localizedDescription().to_string())
+}
+
+fn legacy(id: u32, frame: Rect) -> Result<Screenshot, String> {
+ let bounds = core_graphics::geometry::CGRect::new(
+ &core_graphics::geometry::CGPoint::new(frame.x, frame.y),
+ &core_graphics::geometry::CGSize::new(frame.w, frame.h),
+ );
+ let image =
+ create_image(bounds, kCGWindowListOptionIncludingWindow, id, kCGWindowImageBoundsIgnoreFraming | kCGWindowImageNominalResolution)
+ .ok_or("Window capture returned nothing.")?;
+ encode(&image, frame)
+}
+
+fn encode(image: &CGImage, area: Rect) -> Result<Screenshot, String> {
+ let data = NSMutableData::new();
+ let kind = CFString::from_static_string("public.png");
+ let written = unsafe {
+ let destination = CGImageDestinationCreateWithData(Retained::as_ptr(&data).cast(), kind.as_concrete_TypeRef(), 1, std::ptr::null());
+ if destination.is_null() {
+ return Err("Could not encode the screenshot.".into());
+ }
+ let destination = CFType::wrap_under_create_rule(destination);
+ CGImageDestinationAddImage(destination.as_CFTypeRef(), image.as_ptr().cast(), std::ptr::null());
+ CGImageDestinationFinalize(destination.as_CFTypeRef())
+ };
+ if !written {
+ return Err("Could not encode the screenshot.".into());
+ }
+ let (width, height) = (image.width() as u32, image.height() as u32);
+ Ok(Screenshot {
+ png_base64: base64::engine::general_purpose::STANDARD.encode(data.to_vec()),
+ width,
+ height,
+ pixels_per_point: if area.w > 0.0 { width as f64 / area.w } else { 1.0 },
+ frame: area,
+ })
+}
+
+fn associated(target: Rect, candidates: &[(u32, Rect)], related: &[u32]) -> (Vec<u32>, Rect) {
+ let mut ids = vec![];
+ let mut area = target;
+ for (id, frame) in candidates {
+ if related.contains(id) && !ids.contains(id) {
+ ids.push(*id);
+ area = union(area, *frame);
+ }
+ }
+ (ids, area)
+}
+
+fn union(a: Rect, b: Rect) -> Rect {
+ let (x, y) = (a.x.min(b.x), a.y.min(b.y));
+ Rect { x, y, w: (a.x + a.w).max(b.x + b.w) - x, h: (a.y + a.h).max(b.y + b.h) - y }
+}
+
+fn rect(r: objc2_core_foundation::CGRect) -> Rect {
+ Rect { x: r.origin.x, y: r.origin.y, w: r.size.width, h: r.size.height }
+}
+
+fn cg(r: Rect) -> objc2_core_foundation::CGRect {
+ objc2_core_foundation::CGRect::new(objc2_core_foundation::CGPoint::new(r.x, r.y), objc2_core_foundation::CGSize::new(r.w, r.h))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn r(x: f64, y: f64, w: f64, h: f64) -> Rect {
+ Rect { x, y, w, h }
+ }
+
+ fn tuple(r: Rect) -> (f64, f64, f64, f64) {
+ (r.x, r.y, r.w, r.h)
+ }
+
+ #[test]
+ fn capture_excludes_unrelated_overlapping_windows() {
+ let window = r(100.0, 100.0, 400.0, 300.0);
+ let menu = (1, r(450.0, 150.0, 200.0, 300.0));
+ let submenu = (2, r(650.0, 200.0, 200.0, 100.0));
+ let other_document = (3, r(150.0, 150.0, 600.0, 600.0));
+ let (ids, area) = associated(window, &[menu, other_document, submenu], &[1, 2]);
+ assert_eq!(ids, vec![1, 2]);
+ assert_eq!(tuple(area), (100.0, 100.0, 750.0, 350.0));
+ let (ids, area) = associated(window, &[other_document], &[]);
+ assert!(ids.is_empty());
+ assert_eq!(tuple(area), tuple(window));
+ }
+
+ #[test]
+ fn named_panels_join_without_touching_the_window() {
+ let (ids, area) = associated(r(0.0, 0.0, 100.0, 100.0), &[(7, r(200.0, 0.0, 50.0, 50.0))], &[7]);
+ assert_eq!(ids, vec![7]);
+ assert_eq!(tuple(area), (0.0, 0.0, 250.0, 100.0));
+ }
+
+ #[test]
+ fn unusable_frames_are_refused_before_allocating() {
+ for frame in [
+ r(0.0, 0.0, 32768.0, 32768.0),
+ r(0.0, 0.0, 0.0, 10.0),
+ r(0.0, 0.0, 10.0, 0.2),
+ r(f64::NAN, 0.0, 10.0, 10.0),
+ r(0.0, 0.0, f64::INFINITY, 10.0),
+ r(0.0, 0.0, 1e9, 10.0),
+ r(0.0, 0.0, -5.0, 10.0),
+ ] {
+ assert!(size(frame).is_err(), "{frame:?}");
+ assert!(compose(frame, &[]).is_err(), "{frame:?}");
+ }
+ assert_eq!(size(r(-3.0, -4.0, 10.4, 20.6)), Ok((10, 21)));
+ }
+
+ fn tile(w: usize, h: usize, bands: &[[f64; 3]]) -> CGImage {
+ let space = CGColorSpace::create_with_name(unsafe { kCGColorSpaceSRGB }).unwrap();
+ let context = CGContext::create_bitmap_context(None, w, h, 8, 0, &space, kCGImageAlphaPremultipliedLast);
+ let band = h as f64 / bands.len() as f64;
+ for (i, [red, green, blue]) in bands.iter().enumerate() {
+ context.set_rgb_fill_color(*red, *green, *blue, 1.0);
+ let y = h as f64 - (i as f64 + 1.0) * band;
+ context.fill_rect(core_graphics::geometry::CGRect::new(
+ &core_graphics::geometry::CGPoint::new(0.0, y),
+ &core_graphics::geometry::CGSize::new(w as f64, band),
+ ));
+ }
+ context.create_image().unwrap()
+ }
+
+ fn pixel(image: &CGImage, x: usize, y: usize) -> [u8; 4] {
+ let data = image.data();
+ let i = y * image.bytes_per_row() + x * 4;
+ data.bytes()[i..i + 4].try_into().unwrap()
+ }
+
+ #[test]
+ fn compose_places_tiles_top_down_at_their_screen_positions() {
+ let (green, red, blue) = ([0.0, 1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]);
+ let area = r(-5.0, -20.0, 10.0, 50.0);
+ let pieces = [
+ Piece { image: tile(10, 20, &[green, red]), at: r(-5.0, -20.0, 10.0, 20.0) },
+ Piece { image: tile(10, 20, &[blue]), at: r(-5.0, 10.0, 10.0, 20.0) },
+ ];
+ let image = compose(area, &pieces).unwrap();
+ assert_eq!((image.width(), image.height()), (10, 50));
+ assert_eq!(pixel(&image, 5, 0), [0, 255, 0, 255]);
+ assert_eq!(pixel(&image, 5, 9), [0, 255, 0, 255]);
+ assert_eq!(pixel(&image, 5, 10), [255, 0, 0, 255]);
+ assert_eq!(pixel(&image, 5, 19), [255, 0, 0, 255]);
+ assert_eq!(pixel(&image, 5, 20), [0, 0, 0, 0]);
+ assert_eq!(pixel(&image, 5, 29), [0, 0, 0, 0]);
+ assert_eq!(pixel(&image, 5, 30), [0, 0, 255, 255]);
+ assert_eq!(pixel(&image, 5, 49), [0, 0, 255, 255]);
+ }
+
+ #[test]
+ fn a_lone_window_keeps_its_frame() {
+ let (ids, area) = associated(r(10.0, 20.0, 30.0, 40.0), &[], &[]);
+ assert!(ids.is_empty());
+ assert_eq!(tuple(area), (10.0, 20.0, 30.0, 40.0));
+ }
+}
diff --git a/desktop/desk/src/focus.rs b/desktop/desk/src/focus.rs
new file mode 100644
index 000000000..444a0b1e9
--- /dev/null
+++ b/desktop/desk/src/focus.rs
@@ -0,0 +1,469 @@
+use crate::protocol::{ErrorCode, Failure, Rect};
+use core_foundation::base::{CFRetain, CFType, TCFType};
+use core_foundation::boolean::CFBoolean;
+use core_foundation::dictionary::CFDictionary;
+use core_foundation::number::CFNumber;
+use core_foundation::string::CFString;
+use core_graphics::event::CGEvent;
+use core_graphics::geometry::{CGPoint, CGRect};
+use foreign_types::ForeignType;
+use objc2::msg_send;
+use objc2::rc::autoreleasepool;
+use objc2_app_kit::{NSApplicationActivationOptions, NSEvent, NSEventModifierFlags, NSEventType, NSRunningApplication};
+use objc2_foundation::NSPoint;
+use std::cell::{Cell, RefCell};
+use std::collections::HashMap;
+use std::ffi::c_void;
+use std::rc::{Rc, Weak};
+use std::sync::atomic::{AtomicI32, Ordering};
+use std::time::{Duration, Instant};
+
+static EXPECTED_FOREGROUND: AtomicI32 = AtomicI32::new(0);
+
+fn screen_available() -> bool {
+ let Some(session) = objc2_core_graphics::CGSessionCopyCurrentDictionary() else {
+ return false;
+ };
+ let session = unsafe {
+ CFDictionary::<CFString, CFType>::wrap_under_get_rule(objc2_core_foundation::CFRetained::as_ptr(&session).as_ptr().cast())
+ };
+ let field = |name: &str| session.find(CFString::new(name)).and_then(|v| v.downcast::<CFBoolean>()).map(bool::from);
+ field("CGSSessionScreenIsLocked") != Some(true) && field("kCGSSessionOnConsoleKey") == Some(true)
+}
+
+fn synthetic_focus(pid: i32, window: u32, subtype: i16) -> Result<(), Failure> {
+ autoreleasepool(|_| {
+ let event = NSEvent::otherEventWithType_location_modifierFlags_timestamp_windowNumber_context_subtype_data1_data2(
+ NSEventType::AppKitDefined,
+ NSPoint::new(0.0, 0.0),
+ NSEventModifierFlags(if window == 0 { 0 } else { 0xc0000 }),
+ 0.0,
+ window as isize,
+ None,
+ subtype,
+ 0,
+ 0,
+ )
+ .ok_or_else(|| Failure::new(ErrorCode::ActionFailed, "Cannot create background focus event"))?;
+ let cg: *mut c_void = unsafe { msg_send![&*event, CGEvent] };
+ if cg.is_null() {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Cannot convert background focus event"));
+ }
+ let event = unsafe { CGEvent::from_ptr(CFRetain(cg.cast()) as _) };
+ event.post_to_pid(pid);
+ Ok::<(), Failure>(())
+ })?;
+ Ok(())
+}
+
+fn frontmost() -> Option<i32> {
+ crate::mac::system_focused_pid()
+}
+
+type Update = unsafe extern "C" fn(i32) -> i32;
+
+struct Updates {
+ connection: i32,
+ enable: Update,
+ frozen: bool,
+}
+
+impl Updates {
+ fn freeze() -> Result<Self, Failure> {
+ let lookup = |name: &std::ffi::CStr| {
+ let pointer = unsafe { libc::dlsym(libc::RTLD_DEFAULT, name.as_ptr()) };
+ if pointer.is_null() {
+ Err(Failure::new(ErrorCode::ActionFailed, "WindowServer activation support is unavailable"))
+ } else {
+ Ok(pointer)
+ }
+ };
+ let main = unsafe { std::mem::transmute::<*mut c_void, unsafe extern "C" fn() -> i32>(lookup(c"CGSMainConnectionID")?) };
+ let disable = unsafe { std::mem::transmute::<*mut c_void, Update>(lookup(c"CGSDisableUpdate")?) };
+ let enable = unsafe { std::mem::transmute::<*mut c_void, Update>(lookup(c"CGSReenableUpdate")?) };
+ let connection = unsafe { main() };
+ if connection == 0 || unsafe { disable(connection) } != 0 {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Cannot suspend display updates for activation"));
+ }
+ Ok(Self { connection, enable, frozen: true })
+ }
+
+ fn resume(&mut self) -> Result<(), Failure> {
+ if self.frozen {
+ if unsafe { (self.enable)(self.connection) } != 0 {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Cannot resume display updates after activation"));
+ }
+ self.frozen = false;
+ }
+ Ok(())
+ }
+}
+
+impl Drop for Updates {
+ fn drop(&mut self) {
+ if let Err(error) = self.resume() {
+ eprintln!("{}", error.message);
+ }
+ }
+}
+
+fn activate_window(target: i32, window: u32, geometry: Option<(CGRect, CGPoint)>) -> Result<(), Failure> {
+ let Some((frame, point)) = geometry else {
+ return synthetic_focus(target, 0, 1);
+ };
+ synthetic_focus(target, window, 1)?;
+ autoreleasepool(|_| {
+ let events = [NSEventType::LeftMouseDown, NSEventType::LeftMouseUp]
+ .into_iter()
+ .enumerate()
+ .map(|(index, kind)| {
+ let native =
+ NSEvent::mouseEventWithType_location_modifierFlags_timestamp_windowNumber_context_eventNumber_clickCount_pressure(
+ kind,
+ NSPoint::new(point.x, point.y),
+ NSEventModifierFlags::empty(),
+ 0.0,
+ window as isize,
+ None,
+ index as isize + 1,
+ 1,
+ 1.0,
+ )
+ .ok_or_else(|| Failure::new(ErrorCode::ActionFailed, "Cannot create window activation event"))?;
+ let raw: *mut c_void = unsafe { msg_send![&*native, CGEvent] };
+ if raw.is_null() {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Cannot convert window activation event"));
+ }
+ let event = unsafe { CGEvent::from_ptr(CFRetain(raw.cast()) as _) };
+ objc2_core_graphics::CGEvent::set_location(
+ Some(unsafe { &*event.as_ptr().cast::<objc2_core_graphics::CGEvent>() }),
+ objc2_core_foundation::CGPoint::new(point.x, point.y),
+ );
+ event.set_integer_value_field(3, 0);
+ event.set_integer_value_field(7, 3);
+ event.set_integer_value_field(41, std::process::id() as i64);
+ crate::mac::in_window(
+ &event,
+ window,
+ &Rect { x: frame.origin.x, y: frame.origin.y, w: frame.size.width, h: frame.size.height },
+ point,
+ )?;
+ Ok(event)
+ })
+ .collect::<Result<Vec<_>, Failure>>()?;
+ check_current()?;
+ for event in events {
+ event.post_to_pid(target);
+ }
+ Ok(())
+ })
+}
+
+fn restore_order(target: i32, window: u32, original_window: &CFType) -> Result<(), Failure> {
+ let windows = core_graphics::window::copy_window_info(0x11, 0)
+ .ok_or_else(|| Failure::new(ErrorCode::ActionFailed, "Cannot verify window order after activation"))?;
+ let mut ahead = Vec::new();
+ for value in windows.iter() {
+ let value = unsafe { CFType::wrap_under_get_rule(*value) };
+ let Some(raw) = value.downcast::<CFDictionary>() else { continue };
+ let dict = unsafe { CFDictionary::<CFString, CFType>::wrap_under_get_rule(raw.as_concrete_TypeRef()) };
+ let number =
+ |name: &str| dict.find(CFString::new(name)).and_then(|value| value.downcast::<CFNumber>()).and_then(|value| value.to_i64());
+ if number("kCGWindowNumber") == Some(i64::from(window)) {
+ if ahead.contains(&(i64::from(target), number("kCGWindowLayer"))) {
+ check_current()?;
+ crate::mac::restore_focused_window(original_window)?;
+ }
+ return Ok(());
+ }
+ if let Some(owner) = number("kCGWindowOwnerPID") {
+ ahead.push((owner, number("kCGWindowLayer")));
+ }
+ }
+ Err(Failure::new(ErrorCode::ActionFailed, "Original focused window disappeared during activation"))
+}
+
+fn enforce_window(target: i32, window: u32, original: i32) -> Result<(), Failure> {
+ let original_window = crate::mac::focused_window(original);
+ let geometry = crate::mac::focus_activation_window(target, window);
+ let original_app = NSRunningApplication::runningApplicationWithProcessIdentifier(original)
+ .ok_or_else(|| Failure::new(ErrorCode::ActionFailed, "Original foreground app exited before activation"))?;
+ let transition = crate::intervention::ActivationTransition::begin(target, original)?;
+ let mut updates = Updates::freeze()?;
+ let activation = (|| {
+ check_current()?;
+ activate_window(target, window, geometry)?;
+ let until = Instant::now() + Duration::from_millis(80);
+ while Instant::now() < until {
+ check_current()?;
+ std::thread::sleep(Duration::from_millis(8));
+ }
+ check_current()
+ })();
+ let restoration = (|| {
+ check_current()?;
+ if frontmost() != Some(original) && !original_app.activateWithOptions(NSApplicationActivationOptions::empty()) {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Cannot restore original foreground app after activation"));
+ }
+ let until = Instant::now() + Duration::from_millis(350);
+ while frontmost() != Some(original) && Instant::now() < until {
+ check_current()?;
+ std::thread::sleep(Duration::from_millis(8));
+ }
+ check_current()?;
+ if frontmost() != Some(original) {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Original foreground app did not return after activation"));
+ }
+ if let Some((id, original_window)) = original_window {
+ restore_order(target, id, &original_window)?;
+ }
+ transition.verify_keyboard_owner()?;
+ check_current()
+ })();
+ let resumed = updates.resume();
+ drop(transition);
+ let mut failures = Vec::new();
+ for (phase, result) in [("activation", activation), ("restoration", restoration), ("display resume", resumed)] {
+ if let Err(error) = result {
+ failures.push(format!("{phase} PID {target}: {}", error.message));
+ }
+ }
+ if !failures.is_empty() && frontmost() != Some(target) {
+ if let Err(error) = synthetic_focus(target, 0, 2) {
+ failures.push(format!("deactivation PID {target}: {}", error.message));
+ }
+ }
+ if failures.is_empty() {
+ Ok(())
+ } else {
+ Err(Failure::new(ErrorCode::ActionFailed, failures.join("; ")))
+ }
+}
+
+#[derive(Default)]
+pub struct Enforcers {
+ entries: RefCell<HashMap<i32, crate::intervention::Retained>>,
+}
+
+impl Enforcers {
+ fn ensure(&self, pid: i32) -> Result<std::sync::Arc<crate::intervention::RetainedState>, Failure> {
+ let failure = self.entries.borrow().get(&pid).and_then(|entry| entry.state.check().err());
+ if let Some(mut failure) = failure {
+ if let Err(cleanup) = self.remove(pid, true) {
+ failure.message.push_str(&format!("; cleanup: {}", cleanup.message));
+ }
+ return Err(failure);
+ }
+ let mut entries = self.entries.borrow_mut();
+ if !entries.contains_key(&pid) {
+ if entries.len() >= 32 {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Retained app protection limit reached"));
+ }
+ entries.insert(pid, crate::intervention::Retained::begin(pid)?);
+ }
+ let state = std::sync::Arc::clone(&entries[&pid].state);
+ state.check()?;
+ Ok(state)
+ }
+
+ fn check(&self, pid: i32) -> Result<(), Failure> {
+ self.entries.borrow().get(&pid).map_or(Ok(()), |entry| entry.state.check())
+ }
+
+ pub fn clear(&self) -> Result<(), Failure> {
+ let pids: Vec<_> = self.entries.borrow().keys().copied().collect();
+ let errors: Vec<_> = pids.into_iter().filter_map(|pid| self.remove(pid, true).err().map(|error| error.message)).collect();
+ if errors.is_empty() {
+ Ok(())
+ } else {
+ Err(Failure::new(ErrorCode::ActionFailed, errors.join("; ")))
+ }
+ }
+
+ pub fn remove(&self, pid: i32, same_process: bool) -> Result<(), Failure> {
+ if let Some(retained) = self.entries.borrow_mut().remove(&pid) {
+ let deactivate = same_process && retained.state.same_process() && retained.state.focused() && frontmost() != Some(pid);
+ retained.state.menu(None);
+ let result = if deactivate { synthetic_focus(pid, 0, 2) } else { Ok(()) };
+ drop(retained);
+ result?;
+ }
+ Ok(())
+ }
+}
+
+impl Drop for Enforcers {
+ fn drop(&mut self) {
+ let pids: Vec<_> = self.entries.borrow().keys().copied().collect();
+ for pid in pids {
+ if let Err(error) = self.remove(pid, true) {
+ eprintln!("App protection cleanup failed: {}", error.message);
+ }
+ }
+ }
+}
+
+thread_local! {
+ static ACTIVATION: RefCell<Weak<Activation>> = const { RefCell::new(Weak::new()) };
+}
+
+struct Activation {
+ original: i32,
+ selected: Cell<(i32, u32)>,
+ activated: RefCell<Vec<(i32, u32)>>,
+ retained: Rc<Enforcers>,
+}
+
+impl Activation {
+ fn ensure(&self, target: i32, window: u32, send: impl FnOnce(i32, u32, i16) -> Result<(), Failure>) -> Result<(), Failure> {
+ if target != self.original && !self.activated.borrow().contains(&(target, window)) {
+ if window == 0 {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Background window has no ID; observe again"));
+ }
+ send(target, window, 1)?;
+ let mut activated = self.activated.borrow_mut();
+ if let Some(current) = activated.iter_mut().find(|(pid, _)| *pid == target) {
+ current.1 = window;
+ } else {
+ activated.push((target, window));
+ }
+ }
+ self.selected.set((target, window));
+ Ok(())
+ }
+}
+
+pub fn activate_target(target: i32, window: u32) -> Result<(), Failure> {
+ check_current()?;
+ let activation = ACTIVATION
+ .with(|current| current.borrow().upgrade())
+ .ok_or_else(|| Failure::new(ErrorCode::ActionFailed, "Target activation requires protected input"))?;
+ crate::intervention::include_target(target);
+ check_current()?;
+ let state = activation.retained.ensure(target)?;
+ if !state.ready(window)? {
+ activation.activated.borrow_mut().retain(|(pid, _)| *pid != target);
+ activation.ensure(target, window, |target, window, _| enforce_window(target, window, activation.original))?;
+ state.activated(window);
+ } else {
+ activation.selected.set((target, window));
+ }
+ check_current()
+}
+
+pub fn activate_current() -> Result<(), Failure> {
+ check_current()?;
+ if let Some(activation) = ACTIVATION.with(|current| current.borrow().upgrade()) {
+ let (target, window) = activation.selected.get();
+ activate_target(target, window)?;
+ }
+ check_current()
+}
+
+pub struct Focus {
+ activation: Rc<Activation>,
+ intervention: Option<crate::intervention::Intervention>,
+}
+
+impl Focus {
+ pub fn begin(target: i32, window: u32, retained: Rc<Enforcers>) -> Result<Self, Failure> {
+ if !screen_available() {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Screen is locked or console session unavailable"));
+ }
+ retained.ensure(target)?;
+ let original = frontmost().ok_or_else(|| Failure::new(ErrorCode::ActionFailed, "Cannot identify the foreground app"))?;
+ let keyboard_owner = crate::intervention::keyboard_focus_pid()?;
+ let intervention = crate::intervention::Intervention::begin(target, original, keyboard_owner)?;
+ if keyboard_owner != crate::intervention::keyboard_focus_pid()? || frontmost() != Some(original) {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Focus changed while starting input; stopped"));
+ }
+ let activation =
+ Rc::new(Activation { original, selected: Cell::new((target, window)), activated: RefCell::new(Vec::new()), retained });
+ ACTIVATION.with(|current| *current.borrow_mut() = Rc::downgrade(&activation));
+ let guard = Self { activation, intervention: Some(intervention) };
+ EXPECTED_FOREGROUND.store(original, Ordering::Release);
+ guard.check()?;
+ Ok(guard)
+ }
+
+ pub fn finish(&mut self) -> Result<(), Failure> {
+ EXPECTED_FOREGROUND.store(0, Ordering::Release);
+ ACTIVATION.with(|current| *current.borrow_mut() = Weak::new());
+ self.activation.activated.borrow_mut().clear();
+ self.intervention.take();
+ Ok(())
+ }
+
+ pub fn check(&self) -> Result<(), Failure> {
+ check_current()?;
+ self.activation.retained.check(self.activation.selected.get().0)?;
+ for (pid, _) in self.activation.activated.borrow().iter() {
+ self.activation.retained.check(*pid)?;
+ }
+ if let Some(intervention) = &self.intervention {
+ intervention.check()?;
+ }
+ Ok(())
+ }
+}
+
+pub fn check_current() -> Result<(), Failure> {
+ let expected = EXPECTED_FOREGROUND.load(Ordering::Acquire);
+ if expected == 0 {
+ return Ok(());
+ }
+ if crate::cancellation::cancelled() {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Input cancelled; activation stopped"));
+ }
+ if !screen_available() {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Screen is locked or console session unavailable"));
+ }
+ crate::intervention::check_current()?;
+ if !frontmost().is_some_and(crate::intervention::permits_foreground) {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Foreground app changed during background input; stopped"));
+ }
+ Ok(())
+}
+
+impl Drop for Focus {
+ fn drop(&mut self) {
+ if let Err(error) = self.finish() {
+ eprintln!("Background focus cleanup failed: {}", error.message);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn activation_tracks_effective_owner_and_balanced_cleanup_targets() {
+ let state = Activation { original: 1, selected: Cell::new((41, 7)), activated: RefCell::new(Vec::new()), retained: Rc::default() };
+ assert!(state.activated.borrow().is_empty());
+ let sent = RefCell::new(Vec::new());
+ let send = |pid, window, subtype| {
+ sent.borrow_mut().push((pid, window, subtype));
+ Ok(())
+ };
+ state.ensure(41, 7, send).unwrap();
+ state.ensure(42, 8, send).unwrap();
+ let (pid, window) = state.selected.get();
+ state.ensure(pid, window, |_, _, _| panic!("duplicate activation")).unwrap();
+ state.ensure(41, 9, send).unwrap();
+ assert_eq!(sent.into_inner(), vec![(41, 7, 1), (42, 8, 1), (41, 9, 1)]);
+ assert_eq!(state.selected.get(), (41, 9));
+ assert_eq!(state.activated.take(), vec![(41, 9), (42, 8)]);
+ assert!(state.activated.take().is_empty());
+ }
+
+ #[test]
+ fn failed_target_activation_preserves_previous_target_and_cleanup() {
+ let state = Activation { original: 1, selected: Cell::new((1, 0)), activated: RefCell::new(Vec::new()), retained: Rc::default() };
+ state.ensure(1, 0, |_, _, _| panic!("foreground activation")).unwrap();
+ assert!(state.ensure(41, 0, |_, _, _| panic!("missing window activation")).is_err());
+ assert!(state.ensure(41, 7, |_, _, _| Err(Failure::new(ErrorCode::ActionFailed, "failed"))).is_err());
+ assert_eq!(state.selected.get(), (1, 0));
+ assert!(state.activated.borrow().is_empty());
+ }
+}
diff --git a/desktop/desk/src/focus_protocol.rs b/desktop/desk/src/focus_protocol.rs
new file mode 100644
index 000000000..6fb1637be
--- /dev/null
+++ b/desktop/desk/src/focus_protocol.rs
@@ -0,0 +1,442 @@
+use std::collections::HashSet;
+
+pub const PROCESS_NOTIFICATION: u32 = 21;
+pub const NEW_FRONT: i64 = 2;
+pub const KEY_FOCUS_TAKEN: i64 = 0x4000;
+pub const KEY_FOCUS_RETURNED: i64 = 0x8000;
+pub const KEY_FOCUS_CHANGED: i64 = 0xf102;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Decision {
+ Pass,
+ Suppress,
+ Release(u32),
+ Redirect(i32),
+ Intervened,
+ Stop(&'static str),
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct Lease {
+ thief: i32,
+ victim: i32,
+ released: bool,
+ returned: bool,
+}
+
+pub struct Scope {
+ owner: i32,
+ foreground: i32,
+ keyboard_owner: i32,
+ targets: HashSet<i32>,
+ lease: Option<Lease>,
+ yielded: bool,
+ activation: Option<i32>,
+ keyboard_interrupted: bool,
+}
+
+impl Scope {
+ pub fn new(owner: i32, foreground: i32, keyboard_owner: i32, target: i32) -> Self {
+ Self {
+ owner,
+ foreground,
+ keyboard_owner,
+ targets: HashSet::from([target]),
+ lease: None,
+ yielded: false,
+ activation: None,
+ keyboard_interrupted: false,
+ }
+ }
+
+ pub fn begin_activation(&mut self, target: i32) -> bool {
+ if self.yielded || self.activation.is_some() || !self.targets.contains(&target) {
+ return false;
+ }
+ self.activation = Some(target);
+ true
+ }
+
+ pub fn keyboard_owner(&self) -> i32 {
+ self.keyboard_owner
+ }
+
+ pub fn interrupted(&self) -> bool {
+ self.keyboard_interrupted
+ }
+
+ pub fn end_activation(&mut self) {
+ self.activation = None;
+ if self.keyboard_interrupted {
+ self.yield_to_user();
+ }
+ }
+
+ pub fn permits_foreground(&self, pid: i32) -> bool {
+ !self.yielded && (pid == self.foreground || self.activation == Some(pid))
+ }
+
+ pub fn activation_event(&self, destination: i32, kind: u32, subtype: i64) -> Decision {
+ if !self.yielded
+ && self.activation.is_some_and(|target| destination == target || destination == self.foreground)
+ && (kind == 19 || (kind == 13 && matches!(subtype, 2 | 22 | 23)))
+ {
+ Decision::Suppress
+ } else {
+ Decision::Pass
+ }
+ }
+
+ pub fn include(&mut self, pid: i32) {
+ if pid > 0 && pid != self.foreground && pid != self.keyboard_owner {
+ self.targets.insert(pid);
+ }
+ }
+
+ pub fn yield_to_user(&mut self) {
+ self.yielded = true;
+ self.lease = None;
+ }
+
+ pub fn notification(&mut self, destination: i32, subtype: i64, subject: i32, theft_id: i64) -> Decision {
+ if self.yielded {
+ return Decision::Pass;
+ }
+ match subtype {
+ KEY_FOCUS_TAKEN if destination == self.keyboard_owner && subject != destination && self.targets.contains(&subject) => {
+ if self.lease.is_none_or(|lease| lease.returned) {
+ self.lease = Some(Lease { thief: subject, victim: destination, released: false, returned: false });
+ Decision::Suppress
+ } else {
+ self.yield_to_user();
+ Decision::Stop("Nested target key-focus lease is unsupported; stopped")
+ }
+ }
+ KEY_FOCUS_CHANGED => {
+ if let Some(lease) = self.lease {
+ if subject == lease.thief && !lease.released {
+ if let Ok(id) = u32::try_from(theft_id) {
+ return Decision::Release(id);
+ }
+ self.yield_to_user();
+ return Decision::Stop("Target key-focus notification has no valid theft ID; stopped");
+ }
+ }
+ Decision::Pass
+ }
+ KEY_FOCUS_RETURNED => {
+ if let Some(lease) = self.lease.as_mut() {
+ if destination == lease.victim && lease.released && !lease.returned {
+ lease.returned = true;
+ return Decision::Suppress;
+ }
+ }
+ Decision::Pass
+ }
+ NEW_FRONT => {
+ if subject > 0 && !self.permits_foreground(subject) {
+ self.yield_to_user();
+ return Decision::Stop("System focus changed during protected input; stopped");
+ }
+ Decision::Pass
+ }
+ _ => Decision::Pass,
+ }
+ }
+
+ pub fn released(&mut self, success: bool) {
+ if let Some(lease) = self.lease.as_mut() {
+ lease.released = success;
+ }
+ if !success {
+ self.yield_to_user();
+ }
+ }
+
+ pub fn user_event(&mut self, kind: u32, source: i32, destination: i32) -> Decision {
+ if source == self.owner || self.yielded {
+ return Decision::Pass;
+ }
+ if (matches!(kind, 1 | 3 | 25) && destination != self.keyboard_owner) || (kind == 22 && self.targets.contains(&destination)) {
+ self.yield_to_user();
+ return Decision::Intervened;
+ }
+ if matches!(kind, 10..=12) {
+ let decision = self.keyboard(source, destination);
+ if decision != Decision::Pass {
+ return decision;
+ }
+ if self.targets.contains(&destination) {
+ self.yield_to_user();
+ return Decision::Intervened;
+ }
+ }
+ Decision::Pass
+ }
+
+ pub fn observe_user_event(&mut self, kind: u32, source: i32, destination: i32) -> Decision {
+ if self.activation.is_some() && source != self.owner && !self.yielded {
+ if matches!(kind, 10..=12) {
+ if destination == self.keyboard_owner {
+ self.keyboard_interrupted = true;
+ return Decision::Pass;
+ }
+ if self.activation == Some(destination) {
+ return Decision::Pass;
+ }
+ }
+ self.yield_to_user();
+ return Decision::Intervened;
+ }
+ match self.user_event(kind, source, destination) {
+ Decision::Redirect(_) => Decision::Pass,
+ decision => decision,
+ }
+ }
+
+ pub fn keyboard(&mut self, source: i32, destination: i32) -> Decision {
+ if self.activation == Some(destination) && source != self.owner && !self.yielded {
+ self.keyboard_interrupted = true;
+ return Decision::Redirect(self.keyboard_owner);
+ }
+ if !self.yielded && source != self.owner {
+ if let Some(lease) = self.lease {
+ if destination == lease.thief {
+ return Decision::Redirect(lease.victim);
+ }
+ }
+ }
+ Decision::Pass
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn forwarded_activation_keys_survive_observer_reentry_until_the_pair_finishes() {
+ let mut state = Scope::new(100, 200, 200, 300);
+ assert!(state.begin_activation(300));
+ for kind in [10, 11] {
+ assert_eq!(state.keyboard(0, 300), Decision::Redirect(200));
+ assert_eq!(state.observe_user_event(kind, 0, 300), Decision::Pass);
+ assert_eq!(state.keyboard(0, 200), Decision::Pass);
+ assert_eq!(state.observe_user_event(kind, 0, 200), Decision::Pass);
+ assert!(state.permits_foreground(300));
+ }
+ assert!(state.interrupted());
+ state.end_activation();
+ assert!(!state.permits_foreground(300));
+ assert!(!state.begin_activation(300));
+ }
+
+ #[test]
+ fn activation_forwarding_does_not_hide_unrelated_user_input() {
+ for (kind, destination) in [(1, 200), (1, 300), (10, 400), (22, 300)] {
+ let mut state = Scope::new(100, 200, 200, 300);
+ assert!(state.begin_activation(300));
+ assert_eq!(state.keyboard(0, 300), Decision::Redirect(200));
+ assert_eq!(state.observe_user_event(kind, 0, destination), Decision::Intervened);
+ assert!(!state.permits_foreground(300));
+ }
+ }
+
+ #[test]
+ fn original_owner_keyboard_input_defers_interruption_until_activation_finishes() {
+ for (down, up) in [(200, 300), (300, 200)] {
+ let mut state = Scope::new(100, 200, 200, 300);
+ assert!(state.begin_activation(300));
+ for (kind, destination) in [(10, down), (11, up)] {
+ assert_eq!(state.observe_user_event(kind, 0, destination), Decision::Pass);
+ assert_eq!(state.keyboard(0, destination), if destination == 300 { Decision::Redirect(200) } else { Decision::Pass });
+ assert_eq!(state.observe_user_event(kind, 0, 200), Decision::Pass);
+ }
+ assert!(state.interrupted());
+ assert!(state.permits_foreground(300));
+ state.end_activation();
+ assert!(!state.permits_foreground(300));
+ }
+ }
+
+ #[test]
+ fn activation_preserves_physical_input_and_limits_suppression() {
+ let mut state = Scope::new(1, 2, 2, 3);
+ assert!(state.begin_activation(3));
+ assert_eq!(state.notification(3, NEW_FRONT, 3, 0), Decision::Pass);
+ assert_eq!(state.activation_event(3, 13, 2), Decision::Suppress);
+ assert_eq!(state.activation_event(3, 13, 1), Decision::Pass);
+ assert_eq!(state.activation_event(4, 13, 2), Decision::Pass);
+ assert_eq!(state.observe_user_event(10, 7, 3), Decision::Pass);
+ assert_eq!(state.keyboard(7, 3), Decision::Redirect(2));
+ assert!(state.interrupted());
+ assert!(state.permits_foreground(2));
+ for kind in 10..=12 {
+ assert_eq!(state.observe_user_event(kind, 7, 3), Decision::Pass);
+ assert_eq!(state.keyboard(7, 3), Decision::Redirect(2));
+ }
+ assert_eq!(state.activation_event(3, 13, 2), Decision::Suppress);
+ state.end_activation();
+ assert!(!state.permits_foreground(3));
+ assert_eq!(state.activation_event(3, 13, 2), Decision::Pass);
+ assert!(!state.begin_activation(3));
+ }
+
+ #[test]
+ fn activation_teardown_and_human_pointer_never_restore_focus() {
+ let mut state = Scope::new(1, 2, 2, 3);
+ assert!(state.begin_activation(3));
+ state.end_activation();
+ assert!(!state.permits_foreground(3));
+ assert_eq!(state.activation_event(3, 19, 0), Decision::Pass);
+ assert!(state.begin_activation(3));
+ assert_eq!(state.observe_user_event(1, 7, 4), Decision::Intervened);
+ assert!(!state.permits_foreground(3));
+ assert_eq!(state.activation_event(2, 13, 22), Decision::Pass);
+ }
+
+ fn scope() -> Scope {
+ Scope::new(100, 200, 200, 300)
+ }
+
+ #[test]
+ fn global_observer_preserves_keys_for_process_local_forwarding() {
+ let mut s = scope();
+ s.notification(200, KEY_FOCUS_TAKEN, 300, 0);
+ for kind in 10..=12 {
+ assert_eq!(s.observe_user_event(kind, 0, 300), Decision::Pass);
+ assert_eq!(s.keyboard(0, 300), Decision::Redirect(200));
+ assert_eq!(s.observe_user_event(kind, 0, 200), Decision::Pass);
+ assert_eq!(s.keyboard(0, 200), Decision::Pass);
+ assert_eq!(s.keyboard(100, 300), Decision::Pass);
+ }
+ assert_eq!(s.observe_user_event(1, 0, 300), Decision::Intervened);
+ assert_eq!(s.keyboard(0, 300), Decision::Pass);
+ }
+
+ #[test]
+ fn nonactivating_panel_keeps_keyboard_ownership_separate_from_foreground() {
+ let mut s = Scope::new(100, 200, 250, 300);
+ assert_eq!(s.notification(200, KEY_FOCUS_TAKEN, 300, 0), Decision::Pass);
+ assert_eq!(s.notification(250, KEY_FOCUS_TAKEN, 300, 0), Decision::Suppress);
+ assert_eq!(s.notification(400, KEY_FOCUS_CHANGED, 300, 7), Decision::Release(7));
+ s.released(true);
+ assert_eq!(s.notification(250, KEY_FOCUS_RETURNED, 0, 0), Decision::Suppress);
+ assert_eq!(s.user_event(10, 0, 300), Decision::Redirect(250));
+ assert_eq!(s.user_event(1, 0, 250), Decision::Pass);
+ assert_eq!(s.notification(200, NEW_FRONT, 200, 0), Decision::Pass);
+ assert!(matches!(s.notification(250, NEW_FRONT, 250, 0), Decision::Stop(_)));
+ }
+
+ #[test]
+ fn completed_pairs_allow_the_next_background_action() {
+ let mut s = scope();
+ for id in [7, 8, 9] {
+ assert_eq!(s.notification(200, KEY_FOCUS_TAKEN, 300, 0), Decision::Suppress);
+ assert_eq!(s.notification(400, KEY_FOCUS_CHANGED, 300, id), Decision::Release(id as u32));
+ s.released(true);
+ assert_eq!(s.notification(200, KEY_FOCUS_RETURNED, 0, 0), Decision::Suppress);
+ assert_eq!(s.keyboard(0, 300), Decision::Redirect(200));
+ }
+ }
+
+ #[test]
+ fn nested_leases_stop_and_a_forwarded_key_cannot_loop() {
+ let mut s = scope();
+ s.notification(200, KEY_FOCUS_TAKEN, 300, 0);
+ let Decision::Redirect(destination) = s.user_event(10, 0, 300) else { panic!() };
+ assert_eq!(s.user_event(10, 0, destination), Decision::Pass);
+ assert!(matches!(s.notification(200, KEY_FOCUS_TAKEN, 300, 0), Decision::Stop(_)));
+ assert_eq!(s.keyboard(0, 300), Decision::Pass);
+ }
+
+ #[test]
+ fn app_switching_and_direct_target_input_yield_without_consuming_events() {
+ let mut s = scope();
+ s.notification(200, KEY_FOCUS_TAKEN, 300, 0);
+ assert_eq!(s.user_event(10, 0, 300), Decision::Redirect(200));
+ assert_eq!(s.user_event(11, 0, 300), Decision::Redirect(200));
+ assert_eq!(s.user_event(10, 100, 300), Decision::Pass);
+ assert_eq!(s.user_event(10, 0, 200), Decision::Pass);
+ assert_eq!(s.user_event(1, 0, 300), Decision::Intervened);
+ assert_eq!(s.user_event(10, 0, 300), Decision::Pass);
+ assert_eq!(scope().user_event(1, 0, 300), Decision::Intervened);
+ assert_eq!(scope().user_event(10, 0, 300), Decision::Intervened);
+ assert_eq!(scope().user_event(10, 0, 200), Decision::Pass);
+ }
+
+ #[test]
+ fn physical_scroll_on_target_yields_without_consuming_the_event() {
+ let mut s = scope();
+ s.include(301);
+ assert_eq!(s.user_event(22, 100, 300), Decision::Pass);
+ assert_eq!(s.user_event(22, 0, 200), Decision::Pass);
+ assert_eq!(s.user_event(22, 0, 900), Decision::Pass);
+ assert_eq!(s.user_event(22, 0, 301), Decision::Intervened);
+ assert_eq!(s.notification(200, KEY_FOCUS_TAKEN, 300, 0), Decision::Pass);
+ assert_eq!(scope().user_event(22, 0, 300), Decision::Intervened);
+ }
+
+ #[test]
+ fn release_is_tied_to_an_observed_target_lease() {
+ let mut s = scope();
+ assert_eq!(s.notification(400, KEY_FOCUS_CHANGED, 300, 7), Decision::Pass);
+ assert_eq!(s.notification(200, KEY_FOCUS_TAKEN, 300, 0), Decision::Suppress);
+ assert_eq!(s.notification(400, KEY_FOCUS_CHANGED, 900, 7), Decision::Pass);
+ assert_eq!(s.notification(400, KEY_FOCUS_CHANGED, 300, 7), Decision::Release(7));
+ s.released(true);
+ assert_eq!(s.notification(400, KEY_FOCUS_CHANGED, 300, 7), Decision::Pass);
+ assert_eq!(s.notification(200, KEY_FOCUS_RETURNED, 0, 0), Decision::Suppress);
+ assert_eq!(s.notification(200, KEY_FOCUS_RETURNED, 0, 0), Decision::Pass);
+ }
+
+ #[test]
+ fn only_misrouted_external_keys_are_forwarded() {
+ let mut s = scope();
+ assert_eq!(s.keyboard(0, 300), Decision::Pass);
+ s.notification(200, KEY_FOCUS_TAKEN, 300, 0);
+ assert_eq!(s.keyboard(0, 300), Decision::Redirect(200));
+ assert_eq!(s.keyboard(100, 300), Decision::Pass);
+ assert_eq!(s.keyboard(0, 200), Decision::Pass);
+ assert_eq!(s.keyboard(0, 900), Decision::Pass);
+ s.yield_to_user();
+ assert_eq!(s.keyboard(0, 300), Decision::Pass);
+ }
+
+ #[test]
+ fn unrelated_notifications_pass() {
+ let mut s = scope();
+ for (destination, subject) in [(900, 300), (200, 900)] {
+ assert_eq!(s.notification(destination, KEY_FOCUS_TAKEN, subject, 1), Decision::Pass);
+ }
+ assert_eq!(s.notification(200, 99, 300, 1), Decision::Pass);
+ }
+
+ #[test]
+ fn genuine_activation_is_never_suppressed() {
+ let mut s = scope();
+ s.notification(200, KEY_FOCUS_TAKEN, 300, 0);
+ assert!(matches!(s.notification(200, NEW_FRONT, 300, 0), Decision::Stop(_)));
+ assert_eq!(s.keyboard(0, 300), Decision::Pass);
+ assert_eq!(s.notification(400, KEY_FOCUS_CHANGED, 300, 7), Decision::Pass);
+ }
+
+ #[test]
+ fn delegated_processes_require_explicit_registration() {
+ let mut s = scope();
+ assert_eq!(s.notification(200, KEY_FOCUS_TAKEN, 301, 0), Decision::Pass);
+ s.include(301);
+ assert_eq!(s.notification(200, KEY_FOCUS_TAKEN, 301, 0), Decision::Suppress);
+ assert_eq!(s.keyboard(0, 301), Decision::Redirect(200));
+ }
+
+ #[test]
+ fn invalid_ids_and_failed_release_do_not_repeat() {
+ let mut s = scope();
+ s.notification(200, KEY_FOCUS_TAKEN, 300, 0);
+ assert!(matches!(s.notification(400, KEY_FOCUS_CHANGED, 300, -1), Decision::Stop(_)));
+ assert_eq!(s.notification(400, KEY_FOCUS_CHANGED, 300, i64::MAX), Decision::Pass);
+ s.released(false);
+ assert_eq!(s.keyboard(0, 300), Decision::Pass);
+ }
+}
diff --git a/desktop/desk/src/intervention.rs b/desktop/desk/src/intervention.rs
new file mode 100644
index 000000000..9ec23dba1
--- /dev/null
+++ b/desktop/desk/src/intervention.rs
@@ -0,0 +1,730 @@
+use crate::focus_protocol::{Decision, Scope, PROCESS_NOTIFICATION};
+use crate::protocol::{ErrorCode, Failure};
+use core_foundation::base::TCFType;
+use core_foundation::mach_port::CFMachPort;
+use core_foundation::runloop::{kCFRunLoopDefaultMode, CFRunLoop, CFRunLoopSource};
+use objc2_core_graphics::{
+ CGEvent, CGEventField, CGEventTapLocation, CGEventTapOptions, CGEventTapPlacement, CGEventTapProxy, CGEventType,
+};
+use std::collections::{HashMap, HashSet};
+use std::ffi::c_void;
+use std::ptr::NonNull;
+use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
+use std::sync::{mpsc, Arc, Mutex};
+use std::thread::{self, JoinHandle};
+use std::time::Duration;
+
+type ReleaseKeyFocus = unsafe extern "C" fn(u32) -> i32;
+type GetKeyFocus = unsafe extern "C" fn(*mut ProcessSerialNumber, *mut u8) -> i32;
+type GetProcessPid = unsafe extern "C" fn(*const ProcessSerialNumber, *mut i32) -> i32;
+
+#[link(name = "Carbon", kind = "framework")]
+unsafe extern "C" {
+ fn IsSecureEventInputEnabled() -> u8;
+}
+
+fn check_secure_input() -> Result<(), Failure> {
+ if unsafe { IsSecureEventInputEnabled() } != 0 {
+ return Err(Failure::new(
+ ErrorCode::ActionFailed,
+ "macOS Secure Input is active; keyboard protection is unavailable, so computer input was stopped",
+ ));
+ }
+ Ok(())
+}
+
+#[repr(C)]
+#[derive(Default)]
+struct ProcessSerialNumber {
+ high: u32,
+ low: u32,
+}
+
+pub fn keyboard_focus_pid() -> Result<i32, Failure> {
+ let query = unsafe { libc::dlsym(libc::RTLD_DEFAULT, c"CPSGetKeyFocusProcess".as_ptr()) };
+ let resolve = unsafe { libc::dlsym(libc::RTLD_DEFAULT, c"GetProcessPID".as_ptr()) };
+ if query.is_null() || resolve.is_null() {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Keyboard-focus ownership is unavailable; input was not started"));
+ }
+ let query = unsafe { std::mem::transmute::<*mut c_void, GetKeyFocus>(query) };
+ let resolve = unsafe { std::mem::transmute::<*mut c_void, GetProcessPid>(resolve) };
+ let mut process = ProcessSerialNumber::default();
+ let mut flags = 0;
+ let mut pid = 0;
+ let status = unsafe { query(&mut process, &mut flags) };
+ if status != 0 {
+ return Err(Failure::new(
+ ErrorCode::ActionFailed,
+ format!("CPSGetKeyFocusProcess failed (status {status}); input was not started"),
+ ));
+ }
+ let status = unsafe { resolve(&process, &mut pid) };
+ if status != 0 {
+ return Err(Failure::new(ErrorCode::ActionFailed, format!("GetProcessPID failed (status {status}); input was not started")));
+ }
+ if pid <= 0 {
+ return Err(Failure::new(
+ ErrorCode::ActionFailed,
+ format!("Keyboard-focus lookup returned invalid PID {pid}; input was not started"),
+ ));
+ }
+ Ok(pid)
+}
+
+const NOTIFICATION_SUBTYPE: CGEventField = CGEventField(64);
+const FOCUS_THEFT_ID: CGEventField = CGEventField(71);
+const FOCUS_SUBJECT_PID: CGEventField = CGEventField(73);
+static ACTIVE: Mutex<Option<Arc<Context>>> = Mutex::new(None);
+
+struct Context {
+ scope: Mutex<Scope>,
+ failure: Mutex<Option<String>>,
+ release: ReleaseKeyFocus,
+ registered: Mutex<HashSet<i32>>,
+ registrations: mpsc::Sender<(i32, mpsc::SyncSender<Result<(), String>>)>,
+}
+
+#[derive(Clone, Copy)]
+enum TapKind {
+ Notifications,
+ Intervention,
+ Keyboard(i32),
+ Retained(i32),
+}
+
+struct TapContext {
+ context: Option<Arc<Context>>,
+ retained: Option<Arc<RetainedState>>,
+ kind: TapKind,
+}
+
+struct Tap {
+ port: objc2_core_foundation::CFRetained<objc2_core_foundation::CFMachPort>,
+ source: CFRunLoopSource,
+ runloop: CFRunLoop,
+ _context: Box<TapContext>,
+}
+
+impl Tap {
+ fn new(context: Arc<Context>, kind: TapKind) -> Result<Self, String> {
+ Self::create(TapContext { context: Some(context), retained: None, kind })
+ }
+
+ fn create(callback: TapContext) -> Result<Self, String> {
+ let mut callback = Box::new(callback);
+ let kind = callback.kind;
+ let info = (&mut *callback as *mut TapContext).cast();
+ let keys = (1 << 10) | (1 << 11) | (1 << 12);
+ let port = unsafe {
+ match kind {
+ TapKind::Keyboard(pid) | TapKind::Retained(pid) => CGEvent::tap_create_for_pid(
+ pid,
+ CGEventTapPlacement::TailAppendEventTap,
+ CGEventTapOptions::Default,
+ if matches!(kind, TapKind::Retained(_)) {
+ [1, 2, 6, 3, 4, 7, 25, 26, 27, 13, 19, 20].into_iter().fold(0, |mask, kind| mask | (1 << kind))
+ } else {
+ keys | (1 << 13) | (1 << 19) | (1 << 20)
+ },
+ Some(receive),
+ info,
+ ),
+ _ => CGEvent::tap_create(
+ CGEventTapLocation::AnnotatedSessionEventTap,
+ CGEventTapPlacement::TailAppendEventTap,
+ if matches!(kind, TapKind::Intervention) { CGEventTapOptions::ListenOnly } else { CGEventTapOptions::Default },
+ if matches!(kind, TapKind::Notifications) {
+ 1 << PROCESS_NOTIFICATION
+ } else {
+ keys | (1 << 1) | (1 << 3) | (1 << 25) | (1 << 22)
+ },
+ Some(receive),
+ info,
+ ),
+ }
+ }
+ .ok_or_else(|| "Could not create a focus-protection event tap".to_string())?;
+ let legacy = unsafe { CFMachPort::wrap_under_get_rule((&*port as *const objc2_core_foundation::CFMachPort).cast_mut().cast()) };
+ let source = match legacy.create_runloop_source(0) {
+ Ok(source) => source,
+ Err(()) => {
+ port.invalidate();
+ return Err("Could not create the focus-protection run-loop source".into());
+ }
+ };
+ let runloop = CFRunLoop::get_current();
+ runloop.add_source(&source, unsafe { kCFRunLoopDefaultMode });
+ CGEvent::tap_enable(&port, true);
+ let tap = Self { port, source, runloop, _context: callback };
+ if !CGEvent::tap_is_enabled(&tap.port) {
+ return Err("Focus-protection event tap did not become enabled".into());
+ }
+ Ok(tap)
+ }
+}
+
+impl Drop for Tap {
+ fn drop(&mut self) {
+ CGEvent::tap_enable(&self.port, false);
+ self.runloop.remove_source(&self.source, unsafe { kCFRunLoopDefaultMode });
+ self.port.invalidate();
+ }
+}
+
+impl Context {
+ fn fail(&self, message: impl Into<String>) {
+ if let Ok(mut failure) = self.failure.lock() {
+ if failure.is_none() {
+ *failure = Some(message.into());
+ }
+ }
+ }
+}
+
+unsafe extern "C-unwind" fn receive(_: CGEventTapProxy, kind: CGEventType, event: NonNull<CGEvent>, info: *mut c_void) -> *mut CGEvent {
+ let callback = unsafe { &*(info as *const TapContext) };
+ if let Some(retained) = &callback.retained {
+ return retained.receive(callback.kind, kind, unsafe { event.as_ref() }, event.as_ptr());
+ }
+ let context = callback.context.as_ref().unwrap();
+ let original = event.as_ptr();
+ if kind.0 >= u32::MAX - 1 {
+ context.fail("Focus protection event tap was disabled; stopped");
+ return original;
+ }
+ if context.failure.lock().map(|failure| failure.is_some()).unwrap_or(true) {
+ return original;
+ }
+ let value = unsafe { event.as_ref() };
+ let field = |id| CGEvent::integer_value_field(Some(value), id);
+ let source = i32::try_from(field(CGEventField::EventSourceUnixProcessID)).unwrap_or(-1);
+ let destination = i32::try_from(field(CGEventField::EventTargetUnixProcessID)).unwrap_or(-1);
+ let Ok(mut scope) = context.scope.lock() else {
+ context.fail("Focus protection state is unavailable; stopped");
+ return original;
+ };
+ if crate::cancellation::cancelled() {
+ scope.yield_to_user();
+ }
+ let decision = if matches!(callback.kind, TapKind::Notifications) && kind.0 == PROCESS_NOTIFICATION {
+ scope.notification(
+ destination,
+ field(NOTIFICATION_SUBTYPE),
+ i32::try_from(field(FOCUS_SUBJECT_PID)).unwrap_or(-1),
+ field(FOCUS_THEFT_ID),
+ )
+ } else if matches!(callback.kind, TapKind::Intervention) {
+ scope.observe_user_event(kind.0, source, destination)
+ } else if matches!(callback.kind, TapKind::Keyboard(pid) if pid == destination) {
+ if matches!(kind.0, 10..=12) {
+ scope.keyboard(source, destination)
+ } else {
+ scope.activation_event(destination, kind.0, field(CGEventField(83)))
+ }
+ } else {
+ Decision::Pass
+ };
+ match decision {
+ Decision::Pass => original,
+ Decision::Suppress => std::ptr::null_mut(),
+ Decision::Redirect(pid) => {
+ CGEvent::set_integer_value_field(Some(value), CGEventField::EventTargetUnixProcessID, i64::from(pid));
+ CGEvent::post_to_pid(pid, Some(value));
+ std::ptr::null_mut()
+ }
+ Decision::Release(id) => {
+ let status = unsafe { (context.release)(id) };
+ scope.released(status == 0);
+ if status != 0 {
+ context.fail(format!("Releasing target key-focus lease failed (status {status}); stopped"));
+ }
+ original
+ }
+ Decision::Stop(message) => {
+ context.fail(message);
+ original
+ }
+ Decision::Intervened => {
+ context.fail("User interacted with the target or requested an app switch; stopped");
+ original
+ }
+ }
+}
+
+pub fn include_target(pid: i32) {
+ let context = ACTIVE.lock().ok().and_then(|active| active.clone());
+ let Some(context) = context else { return };
+ if context.registered.lock().map(|registered| registered.contains(&pid)).unwrap_or(false) {
+ return;
+ }
+ let (send, receive) = mpsc::sync_channel(1);
+ if context.registrations.send((pid, send)).is_err() {
+ context.fail("Focus protection stopped before target registration");
+ return;
+ }
+ match receive.recv_timeout(Duration::from_millis(350)) {
+ Ok(Ok(())) => {}
+ Ok(Err(error)) => context.fail(error),
+ Err(_) => context.fail("Focus protection target registration did not complete within 350 ms"),
+ }
+}
+
+pub struct ActivationTransition {
+ context: Arc<Context>,
+}
+
+impl ActivationTransition {
+ pub fn verify_keyboard_owner(&self) -> Result<(), Failure> {
+ let expected = self
+ .context
+ .scope
+ .lock()
+ .map_err(|_| Failure::new(ErrorCode::ActionFailed, "Focus protection state is unavailable"))?
+ .keyboard_owner();
+ if keyboard_focus_pid()? != expected {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Original keyboard owner was not restored after activation"));
+ }
+ Ok(())
+ }
+
+ pub fn begin(target: i32, original: i32) -> Result<Self, Failure> {
+ include_target(target);
+ include_target(original);
+ check_current()?;
+ let context = ACTIVE
+ .lock()
+ .ok()
+ .and_then(|active| active.clone())
+ .ok_or_else(|| Failure::new(ErrorCode::ActionFailed, "Activation protection is unavailable"))?;
+ let allowed = context.scope.lock().map(|mut scope| scope.begin_activation(target)).unwrap_or(false);
+ if !allowed {
+ return Err(Failure::new(
+ ErrorCode::ActionFailed,
+ "Activation protection cannot start after interruption or during another transition",
+ ));
+ }
+ Ok(Self { context })
+ }
+}
+
+impl Drop for ActivationTransition {
+ fn drop(&mut self) {
+ if let Ok(mut scope) = self.context.scope.lock() {
+ scope.end_activation();
+ if scope.interrupted() {
+ self.context.fail("User typed during activation; input was returned to its original owner and automation stopped");
+ }
+ }
+ }
+}
+
+pub fn permits_foreground(pid: i32) -> bool {
+ ACTIVE
+ .lock()
+ .ok()
+ .and_then(|active| active.clone())
+ .and_then(|context| context.scope.lock().ok().map(|scope| scope.permits_foreground(pid)))
+ .unwrap_or(false)
+}
+
+pub struct Intervention {
+ stop: Arc<AtomicBool>,
+ thread: Option<JoinHandle<()>>,
+ context: Arc<Context>,
+}
+
+impl Intervention {
+ pub fn begin(target: i32, foreground: i32, keyboard_owner: i32) -> Result<Self, Failure> {
+ check_secure_input()?;
+ let pointer = unsafe { libc::dlsym(libc::RTLD_DEFAULT, c"CPSReleaseKeyFocusWithID".as_ptr()) };
+ if pointer.is_null() {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Key-focus lease release is unavailable; input was not started"));
+ }
+ let release = unsafe { std::mem::transmute::<*mut c_void, ReleaseKeyFocus>(pointer) };
+ let (registrations, requests) = mpsc::channel();
+ let context = Arc::new(Context {
+ scope: Mutex::new(Scope::new(std::process::id() as i32, foreground, keyboard_owner, target)),
+ failure: Mutex::new(None),
+ release,
+ registered: Mutex::new(HashSet::new()),
+ registrations,
+ });
+ let stop = Arc::new(AtomicBool::new(false));
+ let quit = Arc::clone(&stop);
+ let callback_context = Arc::clone(&context);
+ let (ready_tx, ready_rx) = mpsc::sync_channel(1);
+ let thread = thread::spawn(move || {
+ let mut taps = Vec::new();
+ for kind in [TapKind::Notifications, TapKind::Intervention, TapKind::Keyboard(target)] {
+ match Tap::new(Arc::clone(&callback_context), kind) {
+ Ok(tap) => taps.push(tap),
+ Err(error) => {
+ let _ = ready_tx.send(Err(error));
+ return;
+ }
+ }
+ }
+ if let Ok(mut registered) = callback_context.registered.lock() {
+ registered.insert(target);
+ }
+ if ready_tx.send(Ok(())).is_ok() {
+ while !quit.load(Ordering::Acquire) {
+ while let Ok((pid, acknowledgement)) = requests.try_recv() {
+ let result: Result<(), String> = (|| {
+ let mut registered =
+ callback_context.registered.lock().map_err(|_| "Focus target state is unavailable".to_string())?;
+ if registered.contains(&pid) {
+ return Ok(());
+ }
+ if pid <= 0 || registered.len() >= 32 {
+ return Err("Focus target registration limit reached or PID invalid".into());
+ }
+ let tap = Tap::new(Arc::clone(&callback_context), TapKind::Keyboard(pid))?;
+ callback_context.scope.lock().map_err(|_| "Focus protection state is unavailable".to_string())?.include(pid);
+ registered.insert(pid);
+ taps.push(tap);
+ Ok(())
+ })();
+ if let Err(error) = &result {
+ callback_context.fail(error.clone());
+ }
+ let _ = acknowledgement.send(result);
+ }
+ CFRunLoop::run_in_mode(unsafe { kCFRunLoopDefaultMode }, Duration::from_millis(30), false);
+ }
+ }
+ });
+ let ready = ready_rx.recv_timeout(Duration::from_millis(350));
+ if matches!(ready, Ok(Ok(()))) {
+ if let Ok(mut active) = ACTIVE.lock() {
+ *active = Some(Arc::clone(&context));
+ }
+ return Ok(Self { stop, thread: Some(thread), context });
+ }
+ stop.store(true, Ordering::Release);
+ let _ = thread.join();
+ let message = match ready {
+ Ok(Err(message)) => message,
+ Err(mpsc::RecvTimeoutError::Timeout) => "Focus protection did not start within 350 ms".into(),
+ _ => "Focus protection stopped before initialization".into(),
+ };
+ Err(Failure::new(ErrorCode::ActionFailed, message))
+ }
+
+ pub fn check(&self) -> Result<(), Failure> {
+ let failure = self
+ .context
+ .failure
+ .lock()
+ .map_err(|_| Failure::new(ErrorCode::ActionFailed, "Focus protection state is unavailable; stopped"))?;
+ if let Some(message) = failure.as_ref() {
+ return Err(Failure::new(ErrorCode::ActionFailed, message.clone()));
+ }
+ check_secure_input()
+ }
+}
+
+pub fn check_current() -> Result<(), Failure> {
+ let active = ACTIVE.lock().map_err(|_| Failure::new(ErrorCode::ActionFailed, "Focus protection state is unavailable; stopped"))?;
+ if let Some(context) = active.as_ref() {
+ let failure =
+ context.failure.lock().map_err(|_| Failure::new(ErrorCode::ActionFailed, "Focus protection state is unavailable; stopped"))?;
+ if let Some(message) = failure.as_ref() {
+ return Err(Failure::new(ErrorCode::ActionFailed, message.clone()));
+ }
+ }
+ check_secure_input()
+}
+
+impl Drop for Intervention {
+ fn drop(&mut self) {
+ self.stop.store(true, Ordering::Release);
+ if let Some(thread) = self.thread.take() {
+ let _ = thread.join();
+ }
+ if let Ok(mut active) = ACTIVE.lock() {
+ *active = None;
+ }
+ }
+}
+
+fn process_identity(pid: i32) -> Option<Option<f64>> {
+ objc2::rc::autoreleasepool(|_| {
+ objc2_app_kit::NSRunningApplication::runningApplicationWithProcessIdentifier(pid)
+ .filter(|app| !app.isTerminated())
+ .map(|app| app.launchDate().map(|date| date.timeIntervalSinceReferenceDate()))
+ })
+}
+
+#[derive(Default)]
+struct LocalActivation {
+ focused: bool,
+ window: u32,
+}
+
+impl LocalActivation {
+ fn event(&mut self, kind: u32, subtype: i64) {
+ match (kind, subtype) {
+ (20, _) | (13, 1) => self.focused = true,
+ (19, _) | (13, 2) => self.focused = false,
+ _ => {}
+ }
+ }
+}
+
+pub struct RetainedState {
+ pid: i32,
+ launched: Option<f64>,
+ owners: Mutex<HashMap<i32, Option<f64>>>,
+ activation: Mutex<LocalActivation>,
+ menu: AtomicI32,
+ failure: Mutex<Option<String>>,
+ registrations: mpsc::Sender<(i32, mpsc::SyncSender<Result<(), String>>)>,
+ runloop: Mutex<Option<CFRunLoop>>,
+}
+
+impl RetainedState {
+ fn wake(&self) {
+ if let Ok(runloop) = self.runloop.lock() {
+ if let Some(runloop) = runloop.as_ref() {
+ runloop.stop();
+ }
+ }
+ }
+
+ pub fn same_process(&self) -> bool {
+ objc2::rc::autoreleasepool(|_| {
+ objc2_app_kit::NSRunningApplication::runningApplicationWithProcessIdentifier(self.pid)
+ .filter(|app| !app.isTerminated())
+ .is_some_and(|app| app.launchDate().map(|d| d.timeIntervalSinceReferenceDate()) == self.launched)
+ })
+ }
+
+ pub fn ready(&self, window: u32) -> Result<bool, Failure> {
+ self.check()?;
+ Ok(self.activation.lock().map(|state| state.focused && state.window == window).unwrap_or(false))
+ }
+
+ pub fn activated(&self, window: u32) {
+ if let Ok(mut state) = self.activation.lock() {
+ state.window = window;
+ }
+ }
+
+ pub fn focused(&self) -> bool {
+ self.activation.lock().map(|state| state.focused).unwrap_or(false)
+ }
+
+ pub fn menu(&self, owner: Option<i32>) {
+ self.menu.store(0, Ordering::Release);
+ let Some(owner) = owner else { return };
+ if owner != self.pid {
+ self.menu.store(-owner, Ordering::Release);
+ let (send, _) = mpsc::sync_channel(1);
+ if self.registrations.send((owner, send)).is_err() {
+ if let Ok(mut failure) = self.failure.lock() {
+ failure.get_or_insert_with(|| "Menu protection stopped".into());
+ }
+ }
+ self.wake();
+ } else {
+ self.menu.store(owner, Ordering::Release);
+ }
+ }
+
+ pub fn check(&self) -> Result<(), Failure> {
+ if !self.same_process() {
+ return Err(Failure::new(ErrorCode::ActionFailed, "Retained target process changed; observe again"));
+ }
+ match self.failure.lock() {
+ Ok(error) => error.as_ref().map_or(Ok(()), |e| Err(Failure::new(ErrorCode::ActionFailed, e.clone()))),
+ Err(_) => Err(Failure::new(ErrorCode::ActionFailed, "Retained app protection is unavailable")),
+ }
+ }
+
+ fn receive(&self, tap: TapKind, kind: CGEventType, event: &CGEvent, original: *mut CGEvent) -> *mut CGEvent {
+ if !self.same_process() {
+ return original;
+ }
+ let TapKind::Retained(pid) = tap else {
+ return original;
+ };
+ let identity = process_identity(pid);
+ if identity.is_none() || self.owners.lock().ok().and_then(|owners| owners.get(&pid).copied()) != identity {
+ return original;
+ }
+ if kind.0 >= u32::MAX - 1 {
+ if let Ok(mut failure) = self.failure.lock() {
+ failure.get_or_insert_with(|| "Retained app event tap was disabled".into());
+ }
+ return original;
+ }
+ let destination = CGEvent::integer_value_field(Some(event), CGEventField::EventTargetUnixProcessID) as i32;
+ if destination == self.pid {
+ let subtype = CGEvent::integer_value_field(Some(event), CGEventField(83));
+ let suppressed = ACTIVE.lock().ok().and_then(|active| active.clone()).is_some_and(|context| {
+ context
+ .scope
+ .lock()
+ .map(|scope| {
+ !crate::cancellation::cancelled()
+ && context.failure.lock().map(|failure| failure.is_none()).unwrap_or(false)
+ && scope.activation_event(destination, kind.0, subtype) == Decision::Suppress
+ })
+ .unwrap_or(false)
+ });
+ if !suppressed {
+ if let Ok(mut state) = self.activation.lock() {
+ state.event(kind.0, subtype);
+ }
+ }
+ }
+ let menu = self.menu.load(Ordering::Acquire);
+ if menu > 0
+ && self.failure.lock().map(|failure| failure.is_none()).unwrap_or(false)
+ && !crate::cancellation::cancelled()
+ && matches!(kind.0, 1 | 2 | 6 | 3 | 4 | 7 | 25 | 26 | 27)
+ {
+ let window = CGEvent::integer_value_field(Some(event), CGEventField(51));
+ if let Some((owner, _)) = u32::try_from(window).ok().filter(|id| *id != 0).and_then(crate::mac::input_window_geometry) {
+ if owner != self.pid && owner != menu {
+ return std::ptr::null_mut();
+ }
+ }
+ }
+ original
+ }
+}
+
+fn stop_retained(stop: &AtomicBool, state: &RetainedState, thread: JoinHandle<()>) {
+ stop.store(true, Ordering::Release);
+ while !thread.is_finished() {
+ state.wake();
+ std::thread::sleep(Duration::from_millis(1));
+ }
+ let _ = thread.join();
+}
+
+pub struct Retained {
+ pub state: Arc<RetainedState>,
+ stop: Arc<AtomicBool>,
+ thread: Option<JoinHandle<()>>,
+}
+
+impl Retained {
+ pub fn begin(pid: i32) -> Result<Self, Failure> {
+ let (registrations, requests) = mpsc::channel();
+ let launched = objc2_app_kit::NSRunningApplication::runningApplicationWithProcessIdentifier(pid)
+ .filter(|app| !app.isTerminated())
+ .ok_or_else(|| Failure::new(ErrorCode::ActionFailed, "Target process exited"))?
+ .launchDate()
+ .map(|d| d.timeIntervalSinceReferenceDate());
+ let state = Arc::new(RetainedState {
+ pid,
+ launched,
+ owners: Mutex::new(HashMap::from([(pid, launched)])),
+ activation: Mutex::new(LocalActivation::default()),
+ menu: AtomicI32::new(0),
+ failure: Mutex::new(None),
+ registrations,
+ runloop: Mutex::new(None),
+ });
+ let callback = Arc::clone(&state);
+ let stop = Arc::new(AtomicBool::new(false));
+ let quit = Arc::clone(&stop);
+ let (send, ready) = mpsc::sync_channel(1);
+ let thread = thread::spawn(move || {
+ if let Ok(mut runloop) = callback.runloop.lock() {
+ *runloop = Some(CFRunLoop::get_current());
+ }
+ let create =
+ |pid| Tap::create(TapContext { context: None, retained: Some(Arc::clone(&callback)), kind: TapKind::Retained(pid) });
+ let Some(_observer) = crate::mac::menu_observer(pid, Arc::clone(&callback)) else {
+ let _ = send.send(Err("Cannot observe target menu lifecycle".to_string()));
+ return;
+ };
+ let mut taps = match create(pid) {
+ Ok(tap) => {
+ let _ = send.send(Ok(()));
+ vec![tap]
+ }
+ Err(error) => {
+ let _ = send.send(Err(error));
+ return;
+ }
+ };
+ let mut registered = HashSet::from([pid]);
+ while !quit.load(Ordering::Acquire) && !crate::cancellation::cancelled() && callback.same_process() {
+ while let Ok((owner, ack)) = requests.try_recv() {
+ let result = if registered.contains(&owner) {
+ Ok(())
+ } else if owner <= 0 || registered.len() >= 32 {
+ Err("Menu target registration limit reached".into())
+ } else {
+ process_identity(owner).ok_or_else(|| "Menu target exited".to_string()).and_then(|identity| {
+ create(owner).map(|tap| {
+ if let Ok(mut owners) = callback.owners.lock() {
+ owners.insert(owner, identity);
+ }
+ registered.insert(owner);
+ taps.push(tap);
+ })
+ })
+ };
+ if let Err(error) = &result {
+ if let Ok(mut failure) = callback.failure.lock() {
+ failure.get_or_insert_with(|| error.clone());
+ }
+ } else {
+ let _ = callback.menu.compare_exchange(-owner, owner, Ordering::AcqRel, Ordering::Acquire);
+ }
+ let _ = ack.send(result);
+ }
+ CFRunLoop::run_in_mode(unsafe { kCFRunLoopDefaultMode }, Duration::from_secs(1), true);
+ }
+ if let Ok(mut failure) = callback.failure.lock() {
+ if failure.is_none() {
+ *failure = Some("Retained app protection stopped".into());
+ }
+ }
+ });
+ let result = ready.recv_timeout(Duration::from_millis(350)).map_err(|e| e.to_string()).and_then(|result| result);
+ if let Err(error) = result {
+ stop_retained(&stop, &state, thread);
+ return Err(Failure::new(ErrorCode::ActionFailed, error));
+ }
+ Ok(Self { state, stop, thread: Some(thread) })
+ }
+}
+
+impl Drop for Retained {
+ fn drop(&mut self) {
+ self.state.menu(None);
+ if let Some(thread) = self.thread.take() {
+ stop_retained(&self.stop, &self.state, thread);
+ }
+ }
+}
+
+#[cfg(test)]
+mod retained_tests {
+ use super::*;
+
+ #[test]
+ fn local_focus_survives_command_boundaries_and_tracks_delivered_activation() {
+ let mut state = LocalActivation::default();
+ state.event(13, 1);
+ state.window = 42;
+ assert!(state.focused);
+ state.event(10, 0);
+ state.event(13, 23);
+ assert!(state.focused);
+ assert_eq!(state.window, 42);
+ state.event(19, 0);
+ assert!(!state.focused);
+ state.event(20, 0);
+ assert!(state.focused);
+ state.event(13, 2);
+ assert!(!state.focused);
+ }
+}
diff --git a/desktop/desk/src/keys.rs b/desktop/desk/src/keys.rs
new file mode 100644
index 000000000..33314fa8e
--- /dev/null
+++ b/desktop/desk/src/keys.rs
@@ -0,0 +1,414 @@
+pub const SHIFT: u64 = 1 << 17;
+pub const CONTROL: u64 = 1 << 18;
+pub const OPTION: u64 = 1 << 19;
+pub const COMMAND: u64 = 1 << 20;
+pub const KEYPAD: u64 = 1 << 21;
+pub const FUNCTION: u64 = 1 << 23;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Chord {
+ pub code: u16,
+ pub flags: u64,
+ pub character: Option<char>,
+ pub modifiers: Vec<u16>,
+}
+
+impl Chord {
+ pub fn remap(&mut self, stroke: crate::text::Stroke) {
+ self.code = stroke.code;
+ if !self.modifiers.iter().any(|code| modifier(*code) == Some(SHIFT)) {
+ self.flags &= !SHIFT;
+ }
+ if stroke.shift {
+ self.flags |= SHIFT;
+ }
+ if stroke.option {
+ self.flags |= OPTION;
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Event {
+ pub code: u16,
+ pub flags: u64,
+ pub down: bool,
+ pub modifier: bool,
+ pub release: bool,
+}
+
+#[derive(Default)]
+pub struct Pressed(Vec<u16>);
+
+impl Pressed {
+ pub fn record(&mut self, event: Event) -> bool {
+ if event.release {
+ let Some(at) = self.0.iter().position(|code| *code == event.code) else { return false };
+ self.0.remove(at);
+ } else {
+ self.0.push(event.code);
+ }
+ true
+ }
+}
+
+fn modifier(code: u16) -> Option<u64> {
+ match code {
+ 54 | 55 => Some(COMMAND),
+ 56 | 60 => Some(SHIFT),
+ 58 | 61 => Some(OPTION),
+ 59 | 62 => Some(CONTROL),
+ 63 => Some(FUNCTION),
+ _ => None,
+ }
+}
+
+fn modifier_code(name: &str) -> Option<u16> {
+ match name {
+ "cmd" | "command" | "meta" | "super" | "super_l" | "meta_l" => Some(55),
+ "super_r" | "meta_r" | "rightcommand" => Some(54),
+ "shift" | "shift_l" => Some(56),
+ "shift_r" | "rightshift" => Some(60),
+ "opt" | "option" | "alt" | "alt_l" => Some(58),
+ "alt_r" | "rightoption" => Some(61),
+ "ctrl" | "control" | "control_l" => Some(59),
+ "control_r" | "rightcontrol" => Some(62),
+ "fn" | "function" => Some(63),
+ _ => None,
+ }
+}
+
+pub fn parse(input: &str) -> Option<Chord> {
+ let (mods, last) = match input.rsplit_once('+') {
+ Some((m, "")) => (m.strip_suffix('+').unwrap_or(m), "+"),
+ Some((m, last)) => (m, last),
+ None => ("", input),
+ };
+ let mut flags = 0;
+ let mut modifiers = Vec::new();
+ for name in mods.split('+').filter(|m| !m.is_empty()) {
+ let code = modifier_code(&name.to_ascii_lowercase())?;
+ let flag = modifier(code)?;
+ if flags & flag == 0 {
+ modifiers.push(code);
+ }
+ flags |= flag;
+ }
+ let name = last.to_ascii_lowercase();
+ if let Some(code) = modifier_code(&name) {
+ return Some(Chord { code, flags, character: None, modifiers });
+ }
+ let single = (name.chars().count() == 1).then(|| name.chars().next().unwrap()).or_else(|| symbol(&name));
+ let symbol = single.and_then(|c| "~!@#$%^&*()_+{}|:\"<>?".chars().position(|x| x == c));
+ let normalized = if let Some(i) = symbol {
+ flags |= SHIFT;
+ "`1234567890-=[]\\;',./".chars().nth(i)?.to_string()
+ } else {
+ name.clone()
+ };
+ let code = code(&single.filter(|_| symbol.is_none()).map(|c| c.to_ascii_lowercase().to_string()).unwrap_or(normalized))?;
+ if name.starts_with("kp_") || name.starts_with("keypad") {
+ flags |= KEYPAD;
+ }
+ Some(Chord { code, flags, character: single.filter(|c| c.is_ascii() && !c.is_control()), modifiers })
+}
+
+pub fn events(code: u16, flags: u64) -> Vec<Event> {
+ events_with_modifiers(code, flags, &[])
+}
+
+pub fn events_with_modifiers(code: u16, flags: u64, requested: &[u16]) -> Vec<Event> {
+ let modifiers = [(59, CONTROL), (58, OPTION), (56, SHIFT), (55, COMMAND), (63, FUNCTION)];
+ let key_modifier = modifier(code);
+ let held: Vec<_> = modifiers
+ .into_iter()
+ .filter(|(_, f)| flags & f != 0 && Some(*f) != key_modifier)
+ .map(|(default, flag)| (requested.iter().copied().find(|c| modifier(*c) == Some(flag)).unwrap_or(default), flag))
+ .collect();
+ let mut active = 0;
+ let mut out = Vec::new();
+ for &(code, flag) in &held {
+ active |= flag;
+ out.push(Event { code, flags: active, down: true, modifier: true, release: false });
+ }
+ out.push(Event { code, flags: flags | key_modifier.unwrap_or(0), down: true, modifier: key_modifier.is_some(), release: false });
+ out.push(Event { code, flags: flags & !key_modifier.unwrap_or(0), down: false, modifier: key_modifier.is_some(), release: true });
+ for &(code, flag) in held.iter().rev() {
+ active &= !flag;
+ out.push(Event { code, flags: active, down: false, modifier: true, release: true });
+ }
+ out
+}
+
+const SYMBOLS: &[(&str, char)] = &[
+ ("comma", ','),
+ ("period", '.'),
+ ("slash", '/'),
+ ("backslash", '\\'),
+ ("semicolon", ';'),
+ ("colon", ':'),
+ ("apostrophe", '\''),
+ ("quoteright", '\''),
+ ("quotedbl", '"'),
+ ("grave", '`'),
+ ("quoteleft", '`'),
+ ("asciitilde", '~'),
+ ("minus", '-'),
+ ("underscore", '_'),
+ ("equal", '='),
+ ("plus", '+'),
+ ("bracketleft", '['),
+ ("bracketright", ']'),
+ ("braceleft", '{'),
+ ("braceright", '}'),
+ ("parenleft", '('),
+ ("parenright", ')'),
+ ("less", '<'),
+ ("greater", '>'),
+ ("question", '?'),
+ ("exclam", '!'),
+ ("at", '@'),
+ ("numbersign", '#'),
+ ("dollar", '$'),
+ ("percent", '%'),
+ ("asciicircum", '^'),
+ ("ampersand", '&'),
+ ("asterisk", '*'),
+ ("bar", '|'),
+];
+fn symbol(name: &str) -> Option<char> {
+ SYMBOLS.iter().find(|(n, _)| *n == name).map(|(_, c)| *c)
+}
+
+fn code(k: &str) -> Option<u16> {
+ const KEYS: &[(&str, u16)] = &[
+ ("a", 0),
+ ("s", 1),
+ ("d", 2),
+ ("f", 3),
+ ("h", 4),
+ ("g", 5),
+ ("z", 6),
+ ("x", 7),
+ ("c", 8),
+ ("v", 9),
+ ("b", 11),
+ ("q", 12),
+ ("w", 13),
+ ("e", 14),
+ ("r", 15),
+ ("y", 16),
+ ("t", 17),
+ ("1", 18),
+ ("2", 19),
+ ("3", 20),
+ ("4", 21),
+ ("6", 22),
+ ("5", 23),
+ ("=", 24),
+ ("9", 25),
+ ("7", 26),
+ ("-", 27),
+ ("8", 28),
+ ("0", 29),
+ ("]", 30),
+ ("o", 31),
+ ("u", 32),
+ ("[", 33),
+ ("i", 34),
+ ("p", 35),
+ ("return", 36),
+ ("enter", 36),
+ ("l", 37),
+ ("j", 38),
+ ("'", 39),
+ ("k", 40),
+ (";", 41),
+ ("\\", 42),
+ (",", 43),
+ ("/", 44),
+ ("n", 45),
+ ("m", 46),
+ (".", 47),
+ ("tab", 48),
+ ("space", 49),
+ ("`", 50),
+ ("delete", 51),
+ ("backspace", 51),
+ ("escape", 53),
+ ("esc", 53),
+ ("f5", 96),
+ ("f6", 97),
+ ("f7", 98),
+ ("f3", 99),
+ ("f8", 100),
+ ("f9", 101),
+ ("f11", 103),
+ ("f10", 109),
+ ("f12", 111),
+ ("home", 115),
+ ("pageup", 116),
+ ("forwarddelete", 117),
+ ("f4", 118),
+ ("end", 119),
+ ("f2", 120),
+ ("pagedown", 121),
+ ("f1", 122),
+ ("left", 123),
+ ("right", 124),
+ ("down", 125),
+ ("up", 126),
+ ];
+ let extra = match k {
+ "f13" => 105,
+ "f14" => 107,
+ "f15" => 113,
+ "f16" => 106,
+ "f17" => 64,
+ "f18" => 79,
+ "f19" => 80,
+ "f20" => 90,
+ "kp_decimal" | "kp_separator" | "keypaddecimal" => 65,
+ "kp_multiply" | "keypadmultiply" => 67,
+ "kp_add" | "keypadadd" => 69,
+ "kp_clear" | "clear" | "numlock" | "num_lock" => 71,
+ "kp_divide" | "keypaddivide" => 75,
+ "kp_enter" | "keypadenter" => 76,
+ "kp_subtract" | "keypadsubtract" => 78,
+ "kp_equal" | "keypadequal" => 81,
+ "kp_0" | "kp_insert" | "keypad0" => 82,
+ "kp_1" | "kp_end" | "keypad1" => 83,
+ "kp_2" | "kp_down" | "keypad2" => 84,
+ "kp_3" | "kp_next" | "keypad3" => 85,
+ "kp_4" | "kp_left" | "keypad4" => 86,
+ "kp_5" | "kp_begin" | "keypad5" => 87,
+ "kp_6" | "kp_right" | "keypad6" => 88,
+ "kp_7" | "kp_home" | "keypad7" => 89,
+ "kp_8" | "kp_up" | "keypad8" => 91,
+ "kp_9" | "kp_prior" | "keypad9" => 92,
+ "linefeed" => 36,
+ "prior" | "page_up" => 116,
+ "next" | "page_down" => 121,
+ "forward_delete" | "kp_delete" => 117,
+ "help" | "insert" => 114,
+ "minus" => 27,
+ "equal" => 24,
+ "bracketleft" => 33,
+ "bracketright" => 30,
+ "backslash" => 42,
+ "semicolon" => 41,
+ "apostrophe" => 39,
+ "comma" => 43,
+ "period" => 47,
+ "slash" => 44,
+ "grave" => 50,
+ _ => return KEYS.iter().find(|(n, _)| *n == k).map(|(_, c)| *c),
+ };
+ Some(extra)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn interrupted_chords_release_only_posted_down_events() {
+ let events = events(17, COMMAND | SHIFT);
+ for (downs, expected) in [(0, vec![]), (1, vec![56, 56]), (2, vec![56, 55, 55, 56]), (3, vec![56, 55, 17, 17, 55, 56])] {
+ let mut pressed = Pressed::default();
+ let sent: Vec<_> = events
+ .iter()
+ .filter(|event| !event.release)
+ .take(downs)
+ .chain(events.iter().filter(|event| event.release))
+ .filter(|event| pressed.record(**event))
+ .collect();
+ assert_eq!(sent.iter().map(|event| event.code).collect::<Vec<_>>(), expected);
+ assert_eq!(sent.iter().filter(|event| event.down).count(), sent.iter().filter(|event| !event.down).count());
+ assert!(sent.last().is_none_or(|event| event.flags == 0));
+ }
+ }
+
+ #[test]
+ fn named_key_coverage() {
+ for (key, code) in [("F13", 105), ("f20", 90), ("KP_Enter", 76), ("KP_9", 92), ("Page_Down", 121), ("BackSpace", 51)] {
+ assert_eq!(parse(key).unwrap().code, code);
+ }
+ assert_eq!(parse("KP_Enter").unwrap().flags, KEYPAD);
+ assert!(parse("f21").is_none());
+ assert!(parse("hyper+a").is_none());
+ }
+
+ #[test]
+ fn punctuation_and_modifier_aliases() {
+ assert_eq!(parse("ctrl+alt+Delete").unwrap().flags, CONTROL | OPTION);
+ assert_eq!(parse("cmd++").unwrap(), Chord { code: 24, flags: COMMAND | SHIFT, character: Some('+'), modifiers: vec![55] });
+ assert_eq!(parse("+").unwrap().code, 24);
+ assert_eq!(parse("?").unwrap().code, 44);
+ assert_eq!(parse("cmd+plus").unwrap().flags, COMMAND | SHIFT);
+ assert_eq!(parse("question").unwrap().code, 44);
+ let right = parse("Control_R+a").unwrap();
+ assert_eq!(events_with_modifiers(right.code, right.flags, &right.modifiers)[0].code, 62);
+ assert_eq!(parse("Super_L+Shift_L+t").unwrap().flags, COMMAND | SHIFT);
+ assert_eq!(parse("Control_R").unwrap().code, 62);
+ }
+
+ #[test]
+ fn layout_mapping_replaces_implicit_shift_but_preserves_explicit_modifiers() {
+ let stroke = crate::text::Stroke { code: 30, shift: false, option: false };
+ let mut chord = parse("cmd+plus").unwrap();
+ chord.remap(stroke);
+ let events = events_with_modifiers(chord.code, chord.flags, &chord.modifiers);
+ assert_eq!(events.iter().map(|event| event.code).collect::<Vec<_>>(), [55, 30, 30, 55]);
+ assert_eq!(events.iter().map(|event| event.flags).collect::<Vec<_>>(), [COMMAND, COMMAND, COMMAND, 0]);
+ let mut explicit = parse("cmd+Shift_R+plus").unwrap();
+ explicit.remap(stroke);
+ let events = events_with_modifiers(explicit.code, explicit.flags, &explicit.modifiers);
+ assert_eq!(events[0].code, 60);
+ assert_eq!(events[2].flags, COMMAND | SHIFT);
+ let mut option = parse("cmd+numbersign").unwrap();
+ option.remap(crate::text::Stroke { code: 20, shift: false, option: true });
+ assert_eq!(option.flags, COMMAND | OPTION);
+ }
+
+ #[test]
+ fn letter_key_names_are_case_insensitive_before_layout_mapping() {
+ let layout = std::collections::HashMap::from([
+ ('z', crate::text::Stroke { code: 6, shift: false, option: false }),
+ ('Z', crate::text::Stroke { code: 6, shift: true, option: false }),
+ ]);
+ let emitted = |name| {
+ let mut chord = parse(name).unwrap();
+ chord.remap(layout[&chord.character.unwrap()]);
+ events_with_modifiers(chord.code, chord.flags, &chord.modifiers)
+ };
+ let plain = emitted("cmd+z");
+ assert_eq!(emitted("cmd+Z"), plain);
+ assert_eq!(plain.iter().map(|event| event.code).collect::<Vec<_>>(), [55, 6, 6, 55]);
+ let shifted = emitted("cmd+Shift_R+z");
+ assert_eq!(emitted("cmd+Shift_R+Z"), shifted);
+ assert_eq!(shifted.iter().map(|event| event.code).collect::<Vec<_>>(), [60, 55, 6, 6, 55, 60]);
+ assert_eq!(shifted[2].flags, COMMAND | SHIFT);
+ }
+
+ #[test]
+ fn chords_press_and_release_modifiers_in_reverse_order() {
+ let events = events(17, COMMAND | SHIFT);
+ assert_eq!(events.iter().map(|e| e.code).collect::<Vec<_>>(), [56, 55, 17, 17, 55, 56]);
+ assert_eq!(
+ events.iter().map(|e| e.flags).collect::<Vec<_>>(),
+ [SHIFT, SHIFT | COMMAND, SHIFT | COMMAND, SHIFT | COMMAND, SHIFT, 0]
+ );
+ assert!(events[..2].iter().all(|e| e.modifier && e.down && !e.release));
+ assert!(events[3..].iter().all(|e| e.release));
+ }
+
+ #[test]
+ fn standalone_modifier_is_a_flags_pair() {
+ let events = events(62, 0);
+ assert_eq!(events.len(), 2);
+ assert_eq!(events[0].flags, CONTROL);
+ assert_eq!(events[1].flags, 0);
+ assert!(events.iter().all(|e| e.modifier));
+ }
+}
diff --git a/desktop/desk/src/mac.rs b/desktop/desk/src/mac.rs
index dd0ad5da7..137fe5fa9 100644
--- a/desktop/desk/src/mac.rs
+++ b/desktop/desk/src/mac.rs
@@ -1,34 +1,55 @@
use crate::cancellation;
+
+mod input;
+mod markers;
use crate::protocol::*;
-use base64::Engine;
+use crate::text::{self, Key as TextKey, Stroke};
+use crate::tree::{self, Item, Key, Locator, Props, ROOT};
use core_foundation::array::CFArray;
-use core_foundation::base::{CFType, CFTypeRef, TCFType};
+use core_foundation::attributed_string::{CFAttributedString, CFAttributedStringGetAttributes, CFAttributedStringGetString};
+use core_foundation::base::{CFGetTypeID, CFHash, CFRange, CFType, CFTypeRef, TCFType};
use core_foundation::boolean::CFBoolean;
+use core_foundation::data::CFData;
use core_foundation::dictionary::CFDictionary;
use core_foundation::number::CFNumber;
+use core_foundation::runloop::{kCFRunLoopDefaultMode, CFRunLoop, CFRunLoopRunResult, CFRunLoopSource};
use core_foundation::string::{CFString, CFStringRef};
+use core_foundation::url::CFURL;
use core_graphics::event::{CGEvent, CGEventFlags, CGEventType, CGMouseButton, EventField, ScrollEventUnit};
use core_graphics::event_source::{CGEventSource, CGEventSourceStateID};
-use core_graphics::geometry::{CGPoint, CGRect, CGSize};
-use core_graphics::window::{
- create_image, kCGWindowImageBoundsIgnoreFraming, kCGWindowImageNominalResolution, kCGWindowListOptionIncludingWindow,
-};
+use core_graphics::geometry::{CGPoint, CGSize};
+use core_graphics::window::{copy_window_info, kCGWindowListOptionOnScreenOnly, kCGWindowNumber};
use foreign_types::ForeignType;
-use objc2_app_kit::{NSApplicationActivationPolicy, NSWorkspace};
+use objc2_app_kit::{NSApplicationActivationPolicy, NSFont, NSFontDescriptorSymbolicTraits, NSRunningApplication, NSWorkspace};
+use objc2_core_graphics::{CGPreflightScreenCaptureAccess, CGRequestScreenCaptureAccess};
+use objc2_foundation::{NSBundle, NSString};
use std::collections::{HashMap, HashSet};
use std::ffi::c_void;
-use std::time::Duration;
+use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
+use std::time::{Duration, Instant};
type El = CFType;
#[link(name = "ApplicationServices", kind = "framework")]
extern "C" {
fn AXUIElementCreateApplication(pid: i32) -> CFTypeRef;
+ fn AXUIElementCreateSystemWide() -> CFTypeRef;
+ fn AXUIElementGetPid(element: CFTypeRef, pid: *mut i32) -> i32;
fn AXUIElementCopyAttributeValue(e: CFTypeRef, a: CFStringRef, v: *mut CFTypeRef) -> i32;
+ fn AXUIElementCopyMultipleAttributeValues(e: CFTypeRef, names: CFTypeRef, options: u32, v: *mut CFTypeRef) -> i32;
+ fn AXUIElementCopyParameterizedAttributeNames(e: CFTypeRef, names: *mut CFTypeRef) -> i32;
+ fn AXUIElementCopyParameterizedAttributeValue(e: CFTypeRef, name: CFStringRef, parameter: CFTypeRef, value: *mut CFTypeRef) -> i32;
+ fn AXUIElementGetTypeID() -> usize;
+ fn AXValueGetTypeID() -> usize;
+ fn AXObserverCreate(pid: i32, cb: extern "C" fn(CFTypeRef, CFTypeRef, CFStringRef, *mut c_void), out: *mut CFTypeRef) -> i32;
+ fn AXObserverAddNotification(o: CFTypeRef, e: CFTypeRef, name: CFStringRef, refcon: *mut c_void) -> i32;
+ fn AXObserverGetRunLoopSource(o: CFTypeRef) -> core_foundation::runloop::CFRunLoopSourceRef;
+ fn AXValueGetType(v: CFTypeRef) -> u32;
fn AXUIElementSetAttributeValue(e: CFTypeRef, a: CFStringRef, v: CFTypeRef) -> i32;
fn AXUIElementIsAttributeSettable(e: CFTypeRef, a: CFStringRef, v: *mut bool) -> i32;
fn AXUIElementCopyActionNames(e: CFTypeRef, v: *mut CFTypeRef) -> i32;
fn AXUIElementPerformAction(e: CFTypeRef, a: CFStringRef) -> i32;
+ fn AXUIElementCopyElementAtPosition(app: CFTypeRef, x: f32, y: f32, e: *mut CFTypeRef) -> i32;
fn AXUIElementSetMessagingTimeout(e: CFTypeRef, t: f32) -> i32;
fn AXValueGetValue(v: CFTypeRef, kind: u32, out: *mut c_void) -> bool;
fn AXValueCreate(kind: u32, value: *const c_void) -> CFTypeRef;
@@ -39,40 +60,279 @@ extern "C" {
#[link(name = "CoreGraphics", kind = "framework")]
extern "C" {
- fn CGPreflightScreenCaptureAccess() -> bool;
- fn CGRequestScreenCaptureAccess() -> bool;
fn CGEventSetLocation(e: *mut c_void, p: CGPoint);
}
-#[link(name = "CoreFoundation", kind = "framework")]
+#[link(name = "Carbon", kind = "framework")]
extern "C" {
- fn CFHash(cf: CFTypeRef) -> usize;
+ fn TISCopyCurrentKeyboardLayoutInputSource() -> CFTypeRef;
+ fn TISCopyCurrentASCIICapableKeyboardLayoutInputSource() -> CFTypeRef;
+ fn TISGetInputSourceProperty(source: CFTypeRef, key: CFStringRef) -> CFTypeRef;
+ static kTISPropertyUnicodeKeyLayoutData: CFStringRef;
+ fn LMGetKbdType() -> u8;
+ fn UCKeyTranslate(
+ layout: *const u8,
+ code: u16,
+ action: u16,
+ modifiers: u32,
+ keyboard: u32,
+ options: u32,
+ dead: *mut u32,
+ max: usize,
+ len: *mut usize,
+ out: *mut u16,
+ ) -> i32;
}
-const MAX_NODES: usize = 800;
-const MAX_DEPTH: usize = 60;
+const MAX_TREE_CHARS: usize = 60_000;
+const MAX_VISITS: usize = 6_000;
+const MAX_BUSY_VISITS: usize = 800;
+const MAX_DEPTH: usize = 200;
+const CLIPS: &[&str] = &["AXScrollArea", "AXWebArea"];
const BORING: &[&str] = &["AXShowMenu", "AXScrollToVisible", "AXShowDefaultUI", "AXShowAlternateUI"];
+const SPARSE: &[&str] = &["AXWebArea", "AXGroup", "AXScrollArea", "AXLayoutArea", "AXUnknown"];
+const TRANSIENT: &[&str] = &["AXDialog", "AXSystemDialog", "AXFloatingWindow", "AXSystemFloatingWindow"];
+const POPUPS: &[&str] = &["AXSheet", "AXPopover", "AXDrawer", "AXMenu"];
+const INVALID_ELEMENT: i32 = -25202;
+const FETCH: &[&str] = &[
+ "AXRole",
+ "AXSubrole",
+ "AXTitle",
+ "AXDescription",
+ "AXHelp",
+ "AXValue",
+ "AXVisibleCharacterRange",
+ "AXPlaceholderValue",
+ "AXURL",
+ "AXIdentifier",
+ "AXExpanded",
+ "AXFocused",
+ "AXEnabled",
+ "AXSelected",
+ "AXTitleUIElement",
+ "AXPosition",
+ "AXSize",
+ "AXChildren",
+];
const BUSY: &[&str] = &["AXBusyIndicator", "AXProgressIndicator"];
+const PRESS_MS: u64 = 50;
+const HOVER_MS: u64 = 30;
+static EVENT_NUMBER: AtomicI64 = AtomicI64::new(1);
+const SETTLE: Duration = Duration::from_millis(1000);
+const SETTLE_MIN: Duration = Duration::from_millis(200);
+const SETTLE_QUIET: Duration = Duration::from_millis(300);
+const SETTLE_MAX: Duration = Duration::from_millis(2000);
+const NOTICES: &[&str] = &[
+ "AXCreated",
+ "AXUIElementDestroyed",
+ "AXValueChanged",
+ "AXTitleChanged",
+ "AXLayoutChanged",
+ "AXFocusedUIElementChanged",
+ "AXFocusedWindowChanged",
+ "AXMainWindowChanged",
+ "AXSelectedRowsChanged",
+ "AXRowCountChanged",
+ "AXSelectedChildrenChanged",
+ "AXSelectedTextChanged",
+ "AXMenuOpened",
+ "AXMenuClosed",
+ "AXWindowCreated",
+ "AXLoadComplete",
+];
+static EPOCH: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
+
+fn now_ms() -> i64 {
+ EPOCH.get_or_init(Instant::now).elapsed().as_millis() as i64
+}
+
+#[derive(Default)]
+struct Notices {
+ last: AtomicI64,
+ generation: AtomicU64,
+ menu: Option<std::sync::Arc<crate::intervention::RetainedState>>,
+ pid: i32,
+ menus: std::cell::RefCell<Vec<(El, i32)>>,
+}
+
+extern "C" fn noticed(_: CFTypeRef, element: CFTypeRef, notice: CFStringRef, refcon: *mut c_void) {
+ let n = unsafe { &*(refcon as *const Notices) };
+ n.last.store(now_ms(), Ordering::Relaxed);
+ n.generation.fetch_add(1, Ordering::Relaxed);
+ if let Some(state) = &n.menu {
+ let notice = unsafe { CFString::wrap_under_get_rule(notice) }.to_string();
+ if matches!(notice.as_str(), "AXMenuOpened" | "AXMenuClosed" | "AXUIElementDestroyed") {
+ let element = unsafe { CFType::wrap_under_get_rule(element) };
+ let mut menus = n.menus.borrow_mut();
+ if notice == "AXMenuOpened" {
+ let owner = pid_of(&element).filter(|pid| *pid > 0);
+ if let Some(owner) = owner {
+ if !menus.iter().any(|(menu, _)| menu == &element) && menus.len() < 32 {
+ menus.push((element, owner));
+ }
+ }
+ } else if let Some(index) = menus.iter().position(|(menu, _)| menu == &element) {
+ menus.truncate(index);
+ }
+ state.menu(menus.last().map(|(_, owner)| *owner));
+ }
+ }
+}
+
+#[derive(Clone)]
+struct Metadata {
+ actions: Vec<String>,
+ settable: bool,
+ attributed: bool,
+}
+
+#[derive(Default)]
+struct Cache {
+ generation: u64,
+ entries: usize,
+ map: HashMap<usize, Vec<(El, Metadata)>>,
+}
+
+const MAX_CACHED: usize = 20_000;
+
+impl Cache {
+ fn get(&self, e: &El, hash: usize) -> Option<Metadata> {
+ self.map.get(&hash)?.iter().find(|(k, _)| k == e).map(|(_, metadata)| metadata.clone())
+ }
+
+ fn put(&mut self, e: &El, hash: usize, metadata: Metadata) {
+ if self.entries >= MAX_CACHED {
+ self.map.clear();
+ self.entries = 0;
+ }
+ self.entries += 1;
+ self.map.entry(hash).or_default().push((e.clone(), metadata));
+ }
+}
+
+pub(crate) struct Observer {
+ _observer: CFType,
+ source: CFRunLoopSource,
+ notices: Box<Notices>,
+ cache: Cache,
+}
+
+impl Observer {
+ fn new(pid: i32, ax: &El) -> Option<Observer> {
+ Self::with_notices(pid, ax, NOTICES)
+ }
+
+ fn with_notices(pid: i32, ax: &El, names: &[&str]) -> Option<Observer> {
+ let mut o: CFTypeRef = std::ptr::null();
+ let status = unsafe { AXObserverCreate(pid, noticed, &mut o) };
+ let observer = ax_result(status, o)?;
+ let notices = Box::<Notices>::default();
+ let refcon = &*notices as *const Notices as *mut c_void;
+ let added = names
+ .iter()
+ .filter(|n| unsafe { AXObserverAddNotification(o, ax.as_CFTypeRef(), cfs(n).as_concrete_TypeRef(), refcon) } == 0)
+ .count();
+ if added == 0 {
+ return None;
+ }
+ let source = unsafe { CFRunLoopSource::wrap_under_get_rule(AXObserverGetRunLoopSource(o)) };
+ CFRunLoop::get_current().add_source(&source, unsafe { kCFRunLoopDefaultMode });
+ Some(Observer { _observer: observer, source, notices, cache: Cache::default() })
+ }
+
+ fn drain() {
+ for _ in 0..1000 {
+ if !matches!(CFRunLoop::run_in_mode(unsafe { kCFRunLoopDefaultMode }, Duration::ZERO, true), CFRunLoopRunResult::HandledSource)
+ {
+ return;
+ }
+ }
+ }
+
+ fn take_cache(&mut self) -> Cache {
+ Self::drain();
+ let generation = self.notices.generation.load(Ordering::Relaxed);
+ let cache = std::mem::take(&mut self.cache);
+ if cache.generation == generation {
+ cache
+ } else {
+ Cache { generation, ..Cache::default() }
+ }
+ }
+}
+
+impl Drop for Observer {
+ fn drop(&mut self) {
+ CFRunLoop::get_current().remove_source(&self.source, unsafe { kCFRunLoopDefaultMode });
+ }
+}
struct App {
name: String,
bundle: Option<String>,
pid: i32,
active: bool,
+ launched: Option<f64>,
+ path: Option<std::path::PathBuf>,
}
struct Snapshot {
window: u32,
window_element: El,
- origin: (f64, f64),
+ frame: Rect,
elements: Vec<(u32, El)>,
- lines: HashMap<u32, String>,
- next: u32,
+ popups: Vec<El>,
+ text: HashMap<u32, text::ObservedText>,
+ markers: HashMap<u32, (El, String)>,
+ ancestry: HashMap<u32, Vec<El>>,
+ lines: HashMap<Key, String>,
+ ids: tree::Ids<El>,
+}
+
+impl Snapshot {
+ fn matches_window(&self, element: &El, id: u32, frame: Option<Rect>) -> bool {
+ self.window == id && &self.window_element == element && frame == Some(self.frame)
+ }
+}
+
+struct Enablement {
+ prior: Vec<(&'static str, bool)>,
+ ready: bool,
}
#[derive(Default)]
pub struct Desk {
cache: HashMap<i32, Snapshot>,
+ touched: HashMap<i32, Instant>,
+ enabled: HashMap<i32, Enablement>,
+ observers: HashMap<i32, Observer>,
+ launches: HashMap<i32, Option<f64>>,
+ enforcers: std::rc::Rc<crate::focus::Enforcers>,
+}
+
+impl Drop for Desk {
+ fn drop(&mut self) {
+ for (pid, launched) in &self.launches {
+ let same = NSRunningApplication::runningApplicationWithProcessIdentifier(*pid)
+ .filter(|app| !app.isTerminated())
+ .is_some_and(|app| app.launchDate().map(|d| d.timeIntervalSinceReferenceDate()) == *launched);
+ if let Err(error) = self.enforcers.remove(*pid, same) {
+ eprintln!("App protection cleanup failed: {}", error.message);
+ }
+ }
+ for (pid, enablement) in &self.enabled {
+ let current = NSRunningApplication::runningApplicationWithProcessIdentifier(*pid)
+ .filter(|a| !a.isTerminated())
+ .map(|a| a.launchDate().map(|d| d.timeIntervalSinceReferenceDate()));
+ if current.as_ref() != self.launches.get(pid) {
+ continue;
+ }
+ let ax = app_element(*pid);
+ for (name, was) in &enablement.prior {
+ set_attr(&ax, name, &CFBoolean::from(*was).as_CFType());
+ }
+ }
+ }
}
fn fail(code: ErrorCode, m: impl Into<String>) -> Failure {
@@ -83,16 +343,54 @@ fn cfs(s: &str) -> CFString {
CFString::new(s)
}
+fn ax_result(status: i32, value: CFTypeRef) -> Option<CFType> {
+ if value.is_null() {
+ return None;
+ }
+ let value = unsafe { CFType::wrap_under_create_rule(value) };
+ (status == 0).then_some(value)
+}
+
fn attr(e: &El, name: &str) -> Option<CFType> {
let mut v: CFTypeRef = std::ptr::null();
let err = unsafe { AXUIElementCopyAttributeValue(e.as_CFTypeRef(), cfs(name).as_concrete_TypeRef(), &mut v) };
- (err == 0 && !v.is_null()).then(|| unsafe { CFType::wrap_under_create_rule(v) })
+ ax_result(err, v)
+}
+
+fn dead(e: &El) -> bool {
+ let mut v: CFTypeRef = std::ptr::null();
+ let err = unsafe { AXUIElementCopyAttributeValue(e.as_CFTypeRef(), cfs("AXRole").as_concrete_TypeRef(), &mut v) };
+ if !v.is_null() {
+ drop(unsafe { CFType::wrap_under_create_rule(v) });
+ }
+ err == INVALID_ELEMENT
+}
+
+fn app_element(pid: i32) -> El {
+ unsafe { CFType::wrap_under_create_rule(AXUIElementCreateApplication(pid)) }
+}
+
+fn pid_of(element: &El) -> Option<i32> {
+ let mut pid = 0;
+ (unsafe { AXUIElementGetPid(element.as_CFTypeRef(), &mut pid) } == 0 && pid > 0).then_some(pid)
+}
+
+pub(crate) fn system_focused_pid() -> Option<i32> {
+ let system = unsafe { AXUIElementCreateSystemWide() };
+ if system.is_null() {
+ return None;
+ }
+ let system = unsafe { CFType::wrap_under_create_rule(system) };
+ pid_of(&attr(&system, "AXFocusedApplication")?)
}
fn text_of(v: &CFType) -> Option<String> {
if let Some(s) = v.downcast::<CFString>() {
return Some(s.to_string());
}
+ if let Some(url) = v.downcast::<CFURL>() {
+ return Some(url.get_string().to_string());
+ }
let n = v.downcast::<CFNumber>()?;
n.to_i64().map(|i| i.to_string()).or_else(|| n.to_f64().map(|f| f.to_string()))
}
@@ -128,42 +426,38 @@ fn list(v: CFType) -> Vec<CFType> {
fn actions(e: &El) -> Vec<String> {
let mut v: CFTypeRef = std::ptr::null();
- if unsafe { AXUIElementCopyActionNames(e.as_CFTypeRef(), &mut v) } != 0 || v.is_null() {
- return vec![];
- }
- list(unsafe { CFType::wrap_under_create_rule(v) }).iter().filter_map(text_of).collect()
+ let status = unsafe { AXUIElementCopyActionNames(e.as_CFTypeRef(), &mut v) };
+ ax_result(status, v).map(list).unwrap_or_default().iter().filter_map(text_of).collect()
}
fn set_attr(e: &El, name: &str, v: &CFType) -> i32 {
unsafe { AXUIElementSetAttributeValue(e.as_CFTypeRef(), cfs(name).as_concrete_TypeRef(), v.as_CFTypeRef()) }
}
-fn perform(e: &El, action: &str) -> Result<(), Failure> {
+fn perform(host: &El, e: &El, action: &str) -> Result<(), Failure> {
+ input::register_action_target(host, e)?;
match unsafe { AXUIElementPerformAction(e.as_CFTypeRef(), cfs(action).as_concrete_TypeRef()) } {
0 => Ok(()),
n => Err(fail(ErrorCode::ActionFailed, format!("{action} failed (AXError {n})"))),
}
}
-fn clip(s: &str, n: usize) -> String {
- let s = s.replace('\n', "⏎");
- if s.chars().count() > n {
- format!("{}…", s.chars().take(n).collect::<String>())
- } else {
- s
- }
+fn app_policy_allowed(policy: NSApplicationActivationPolicy, include_accessory: bool) -> bool {
+ policy == NSApplicationActivationPolicy::Regular || (include_accessory && policy == NSApplicationActivationPolicy::Accessory)
}
-fn running() -> Vec<App> {
+fn running(include_accessory: bool) -> Vec<App> {
NSWorkspace::sharedWorkspace()
.runningApplications()
.iter()
- .filter(|a| a.activationPolicy() == NSApplicationActivationPolicy::Regular)
+ .filter(|a| app_policy_allowed(a.activationPolicy(), include_accessory))
.map(|a| App {
name: a.localizedName().map(|s| s.to_string()).unwrap_or_default(),
bundle: a.bundleIdentifier().map(|s| s.to_string()),
pid: a.processIdentifier(),
active: a.isActive(),
+ launched: a.launchDate().map(|d| d.timeIntervalSinceReferenceDate()),
+ path: a.bundleURL().and_then(|url| url.path()).and_then(|path| std::fs::canonicalize(path.to_string()).ok()),
})
.collect()
}
@@ -177,21 +471,78 @@ fn source() -> Result<CGEventSource, Failure> {
}
fn post(e: CGEvent, pid: i32) {
+ e.set_integer_value_field(EventField::EVENT_SOURCE_UNIX_PROCESS_ID, std::process::id() as i64);
e.post_to_pid(pid);
std::thread::sleep(Duration::from_millis(8));
}
-fn mouse(pid: i32, win: u32, ty: CGEventType, p: CGPoint, b: CGMouseButton, clicks: i64, flags: CGEventFlags) -> Result<(), Failure> {
+const EVENT_WINDOW_ID: u32 = 51;
+
+pub(crate) fn in_window(e: &CGEvent, window: u32, frame: &Rect, p: CGPoint) -> Result<(), Failure> {
+ let pointer = unsafe { libc::dlsym(libc::RTLD_DEFAULT, c"CGEventSetWindowLocation".as_ptr()) };
+ if pointer.is_null() {
+ return Err(fail(ErrorCode::ActionFailed, "Window-relative event delivery is unavailable"));
+ }
+ let set_location: unsafe extern "C" fn(*mut c_void, CGPoint) = unsafe { std::mem::transmute(pointer) };
+ e.set_integer_value_field(EVENT_WINDOW_ID, window as i64);
+ e.set_integer_value_field(EventField::MOUSE_EVENT_WINDOW_UNDER_MOUSE_POINTER, window as i64);
+ e.set_integer_value_field(EventField::MOUSE_EVENT_WINDOW_UNDER_MOUSE_POINTER_THAT_CAN_HANDLE_THIS_EVENT, window as i64);
+ unsafe { set_location(e.as_ptr().cast(), pt(p.x - frame.x, p.y - frame.y)) };
+ Ok(())
+}
+
+struct Press {
+ host: El,
+ owner: i32,
+ window: El,
+ win: u32,
+ frame: Rect,
+ number: i64,
+}
+
+fn validate_input_window(at: &Press) -> Result<(), Failure> {
+ if input::resolved_window(&at.host, &at.window).ok() != Some((at.owner, at.win, at.frame)) {
+ return Err(fail(ErrorCode::ActionFailed, "The input window moved, resized or closed; observe again before sending input"));
+ }
+ Ok(())
+}
+
+fn mouse(pid: i32, at: &Press, ty: CGEventType, p: CGPoint, b: CGMouseButton, clicks: i64, flags: CGEventFlags) -> Result<(), Failure> {
+ let release = matches!(ty, CGEventType::LeftMouseUp | CGEventType::RightMouseUp | CGEventType::OtherMouseUp);
+ if !release {
+ crate::intervention::include_target(pid);
+ active()?;
+ validate_input_window(at)?;
+ crate::focus::activate_target(at.owner, at.win)?;
+ }
let e = CGEvent::new_mouse_event(source()?, ty, p, b).map_err(|_| fail(ErrorCode::ActionFailed, "mouse event"))?;
e.set_flags(flags);
e.set_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE, clicks);
- e.set_integer_value_field(91, win as i64);
- e.set_integer_value_field(92, win as i64);
+ e.set_integer_value_field(EventField::MOUSE_EVENT_NUMBER, at.number);
+ in_window(&e, at.win, &at.frame, p)?;
+ if !release {
+ active()?;
+ }
post(e, pid);
Ok(())
}
-fn click_at(pid: i32, win: u32, p: CGPoint, button: Button, count: u8, mods: &[Modifier], hold: u32) -> Result<(), Failure> {
+fn prepare_window(pid: i32, host: &El, window: El, win: u32, p: CGPoint) -> Result<Press, Failure> {
+ let (owner, frame) = input::window_geometry(win)
+ .filter(|(owner, _)| *owner == pid)
+ .ok_or_else(|| fail(ErrorCode::NoWindow, "Input window ownership is unavailable; observe again"))?;
+ let press = Press { host: host.clone(), owner, window, win, frame, number: EVENT_NUMBER.fetch_add(1, Ordering::Relaxed) };
+ mouse(pid, &press, CGEventType::MouseMoved, p, CGMouseButton::Left, 0, CGEventFlags::empty())?;
+ cancellation::wait(Duration::from_millis(HOVER_MS));
+ Ok(press)
+}
+
+fn click_at(pid: i32, press: Press, p: CGPoint, button: Button, count: u8, mods: &[Modifier], hold: u32) -> Result<(), Failure> {
+ with_modifiers(pid, flags(mods), || click_at_inner(pid, press, p, button, count, mods, hold))
+}
+
+fn click_at_inner(pid: i32, mut press: Press, p: CGPoint, button: Button, count: u8, mods: &[Modifier], hold: u32) -> Result<(), Failure> {
+ let hold = if hold == 0 { PRESS_MS } else { hold.min(10_000) as u64 };
let (down, up, b) = match button {
Button::Left => (CGEventType::LeftMouseDown, CGEventType::LeftMouseUp, CGMouseButton::Left),
Button::Right => (CGEventType::RightMouseDown, CGEventType::RightMouseUp, CGMouseButton::Right),
@@ -199,139 +550,210 @@ fn click_at(pid: i32, win: u32, p: CGPoint, button: Button, count: u8, mods: &[M
};
for n in 1..=count.max(1) as i64 {
active()?;
- mouse(pid, win, down, p, b, n, flags(mods))?;
- cancellation::wait(Duration::from_millis(hold.min(10_000) as u64));
- mouse(pid, win, up, p, b, n, flags(mods))?;
+ if n > 1 {
+ press.number = EVENT_NUMBER.fetch_add(1, Ordering::Relaxed);
+ }
+ mouse(pid, &press, down, p, b, n, flags(mods))?;
+ cancellation::wait(Duration::from_millis(hold));
+ mouse(pid, &press, up, p, b, n, flags(mods))?;
}
Ok(())
}
fn key(pid: i32, code: u16, flags: CGEventFlags, text: Option<&str>, hold: u32) -> Result<(), Failure> {
+ send_key_events(pid, crate::keys::events(code, flags.bits()), text, hold)
+}
+
+fn send_key_events(pid: i32, steps: Vec<crate::keys::Event>, text: Option<&str>, hold: u32) -> Result<(), Failure> {
+ crate::intervention::include_target(pid);
active()?;
- for down in [true, false] {
- if !down {
+ let prepared = steps.into_iter().map(|step| key_event(step, text)).collect::<Result<Vec<_>, _>>()?;
+ if prepared.iter().any(|(step, _)| !step.release) {
+ crate::focus::activate_current()?;
+ }
+ let mut failure = None;
+ let mut releasing = false;
+ let mut pressed = crate::keys::Pressed::default();
+ for (step, e) in prepared {
+ if step.release && !releasing {
cancellation::wait(Duration::from_millis(hold.min(10_000) as u64));
+ releasing = true;
+ }
+ if step.release && failure.is_none() {
+ failure = active().err();
+ }
+ if !step.release {
+ if failure.is_none() {
+ failure = active().err();
+ }
+ if failure.is_some() {
+ continue;
+ }
+ }
+ if pressed.record(step) {
+ post(e, pid);
}
- let e = CGEvent::new_keyboard_event(source()?, code, down).map_err(|_| fail(ErrorCode::ActionFailed, "key event"))?;
- e.set_flags(flags);
+ }
+ failure.map_or(Ok(()), Err)
+}
+
+fn key_event(step: crate::keys::Event, text: Option<&str>) -> Result<(crate::keys::Event, CGEvent), Failure> {
+ let e = CGEvent::new_keyboard_event(source()?, step.code, step.down).map_err(|_| fail(ErrorCode::ActionFailed, "key event"))?;
+ if step.modifier {
+ e.set_type(CGEventType::FlagsChanged);
+ }
+ e.set_flags(CGEventFlags::from_bits_truncate(step.flags));
+ if !step.modifier {
if let Some(t) = text {
e.set_string(t);
}
- post(e, pid);
+ }
+ Ok((step, e))
+}
+
+fn with_modifiers(pid: i32, flags: CGEventFlags, action: impl FnOnce() -> Result<(), Failure>) -> Result<(), Failure> {
+ crate::intervention::include_target(pid);
+ active()?;
+ let prepared = crate::keys::events(0, flags.bits())
+ .into_iter()
+ .filter(|e| e.modifier)
+ .map(|step| key_event(step, None))
+ .collect::<Result<Vec<_>, _>>()?;
+ let (down, up): (Vec<_>, Vec<_>) = prepared.into_iter().partition(|(step, _)| !step.release);
+ let mut pressed = crate::keys::Pressed::default();
+ let result = (|| {
+ for (step, event) in down {
+ crate::focus::activate_current()?;
+ active()?;
+ pressed.record(step);
+ post(event, pid);
+ }
+ action()
+ })();
+ for (step, event) in up {
+ if pressed.record(step) {
+ post(event, pid);
+ }
+ }
+ result
+}
+
+fn layout_data(source: CFTypeRef) -> Option<CFData> {
+ if source.is_null() {
+ return None;
+ }
+ let source = unsafe { CFType::wrap_under_create_rule(source) };
+ let data = unsafe { TISGetInputSourceProperty(source.as_CFTypeRef(), kTISPropertyUnicodeKeyLayoutData) };
+ (!data.is_null()).then(|| unsafe { CFData::wrap_under_get_rule(data as _) })
+}
+
+fn keyboard() -> HashMap<char, Stroke> {
+ keyboard_for_flags(CGEventFlags::empty())
+}
+
+fn keyboard_for_flags(flags: CGEventFlags) -> HashMap<char, Stroke> {
+ let Some(data) = layout_data(unsafe { TISCopyCurrentKeyboardLayoutInputSource() })
+ .or_else(|| layout_data(unsafe { TISCopyCurrentASCIICapableKeyboardLayoutInputSource() }))
+ else {
+ return HashMap::new();
+ };
+ let kind = unsafe { LMGetKbdType() } as u32;
+ text::layout(|code, shift, option| {
+ let modifiers =
+ if shift { 2 } else { 0 } | if option { 8 } else { 0 } | if flags.contains(CGEventFlags::CGEventFlagCommand) { 1 } else { 0 };
+ let (mut dead, mut len, mut out) = (0u32, 0usize, [0u16; 4]);
+ let status =
+ unsafe { UCKeyTranslate(data.bytes().as_ptr(), code, 0, modifiers, kind, 0, &mut dead, out.len(), &mut len, out.as_mut_ptr()) };
+ if status != 0 || dead != 0 {
+ return None;
+ }
+ let mut chars = char::decode_utf16(out[..len].iter().copied());
+ match (chars.next(), chars.next()) {
+ (Some(Ok(c)), None) => Some(c),
+ _ => None,
+ }
+ })
+}
+
+fn type_text(pid: i32, s: &str) -> Result<(), Failure> {
+ for k in text::plan(s, &keyboard()) {
+ match k {
+ TextKey::Return => key(pid, chord("return")?.0, CGEventFlags::empty(), None, 0)?,
+ TextKey::Tab => key(pid, chord("tab")?.0, CGEventFlags::empty(), None, 0)?,
+ TextKey::Mapped(c, stroke) => {
+ let mut mods = vec![];
+ if stroke.shift {
+ mods.push(Modifier::Shift);
+ }
+ if stroke.option {
+ mods.push(Modifier::Option);
+ }
+ key(pid, stroke.code, flags(&mods), Some(&c.to_string()), 0)?
+ }
+ TextKey::Unicode(c) => key(pid, 0, CGEventFlags::empty(), Some(&c.to_string()), 0)?,
+ }
}
Ok(())
}
+struct Field {
+ element: El,
+ value: String,
+ selection: Option<(usize, usize)>,
+}
+
+fn field(ax: &El) -> Option<Field> {
+ let element = attr(ax, "AXFocusedUIElement")?;
+ let value = attr(&element, "AXValue")?.downcast::<CFString>()?.to_string();
+ let selection = ax_value::<[isize; 2]>(&element, "AXSelectedTextRange", 4)
+ .filter(|[start, len]| *start >= 0 && *len >= 0)
+ .map(|[start, len]| (start as usize, len as usize));
+ Some(Field { element, value, selection })
+}
+
+fn paste(pid: i32, ax: &El, s: &str, format: PasteFormat) -> Result<(), Failure> {
+ let v = match keyboard_for_flags(CGEventFlags::CGEventFlagCommand).get(&'v') {
+ Some(k) => k.code,
+ None => chord("v")?.0,
+ };
+ let before = field(ax);
+ let pasted = text::expected(s, format);
+ let mut check = || {
+ use crate::pasteboard::Check;
+ match (&before, field(ax)) {
+ (Some(b), Some(now))
+ if now.element == b.element && pasted.iter().any(|p| text::inserted(&b.value, b.selection, &now.value, p)) =>
+ {
+ Check::Landed
+ }
+ (Some(_), Some(_)) => Check::NotYet,
+ _ => Check::Unreadable,
+ }
+ };
+ let cmd_v = || key(pid, v, CGEventFlags::CGEventFlagCommand, None, 0);
+ crate::pasteboard::paste(s, format, cmd_v, &mut check, &active)
+}
+
fn active() -> Result<(), Failure> {
if cancellation::cancelled() {
Err(fail(ErrorCode::ActionFailed, "Computer action cancelled"))
} else {
- Ok(())
+ crate::focus::check_current()
}
}
-fn keycode(k: &str) -> Option<u16> {
- const KEYS: &[(&str, u16)] = &[
- ("a", 0),
- ("s", 1),
- ("d", 2),
- ("f", 3),
- ("h", 4),
- ("g", 5),
- ("z", 6),
- ("x", 7),
- ("c", 8),
- ("v", 9),
- ("b", 11),
- ("q", 12),
- ("w", 13),
- ("e", 14),
- ("r", 15),
- ("y", 16),
- ("t", 17),
- ("1", 18),
- ("2", 19),
- ("3", 20),
- ("4", 21),
- ("6", 22),
- ("5", 23),
- ("=", 24),
- ("9", 25),
- ("7", 26),
- ("-", 27),
- ("8", 28),
- ("0", 29),
- ("]", 30),
- ("o", 31),
- ("u", 32),
- ("[", 33),
- ("i", 34),
- ("p", 35),
- ("return", 36),
- ("enter", 36),
- ("l", 37),
- ("j", 38),
- ("'", 39),
- ("k", 40),
- (";", 41),
- ("\\", 42),
- (",", 43),
- ("/", 44),
- ("n", 45),
- ("m", 46),
- (".", 47),
- ("tab", 48),
- ("space", 49),
- ("`", 50),
- ("delete", 51),
- ("backspace", 51),
- ("escape", 53),
- ("esc", 53),
- ("f5", 96),
- ("f6", 97),
- ("f7", 98),
- ("f3", 99),
- ("f8", 100),
- ("f9", 101),
- ("f11", 103),
- ("f10", 109),
- ("f12", 111),
- ("home", 115),
- ("pageup", 116),
- ("forwarddelete", 117),
- ("f4", 118),
- ("end", 119),
- ("f2", 120),
- ("pagedown", 121),
- ("f1", 122),
- ("left", 123),
- ("right", 124),
- ("down", 125),
- ("up", 126),
- ];
- let k = k.to_ascii_lowercase();
- KEYS.iter().find(|(n, _)| *n == k).map(|(_, c)| *c)
-}
-
fn chord(k: &str) -> Result<(u16, Vec<Modifier>), Failure> {
- let unknown = || fail(ErrorCode::UnknownKey, format!("unknown key {k}"));
- let (mods, last) = match k.rsplit_once('+') {
- Some((m, "")) if !m.is_empty() => (m.strip_suffix('+').unwrap_or(m), "+"),
- Some((m, last)) => (m, last),
- None => ("", k),
- };
- let mods = mods
- .split('+')
- .filter(|m| !m.is_empty())
- .map(|m| match m.to_ascii_lowercase().as_str() {
- "cmd" | "command" | "meta" | "super" => Ok(Modifier::Command),
- "shift" => Ok(Modifier::Shift),
- "opt" | "option" | "alt" => Ok(Modifier::Option),
- "ctrl" | "control" => Ok(Modifier::Control),
- _ => Err(unknown()),
- })
- .collect::<Result<Vec<_>, _>>()?;
- Ok((keycode(last).ok_or_else(unknown)?, mods))
+ let parsed = crate::keys::parse(k).ok_or_else(|| fail(ErrorCode::UnknownKey, format!("unknown key {k}")))?;
+ let mods = [
+ (crate::keys::COMMAND, Modifier::Command),
+ (crate::keys::SHIFT, Modifier::Shift),
+ (crate::keys::OPTION, Modifier::Option),
+ (crate::keys::CONTROL, Modifier::Control),
+ ]
+ .into_iter()
+ .filter_map(|(bit, m)| (parsed.flags & bit != 0).then_some(m))
+ .collect();
+ Ok((parsed.code, mods))
}
fn flags(mods: &[Modifier]) -> CGEventFlags {
@@ -349,135 +771,695 @@ fn center(w: &El) -> CGPoint {
frame(w).map(|f| pt(f.x + f.w / 2.0, f.y + f.h / 2.0)).unwrap_or(pt(0.0, 0.0))
}
+fn primary_window<T>(focused: Option<T>, main: impl FnOnce() -> Option<T>, windows: impl FnOnce() -> Vec<T>) -> Option<T> {
+ focused.or_else(main).or_else(|| windows().pop())
+}
+
fn window_of(app: &El) -> Option<El> {
- attr(app, "AXFocusedWindow")
- .or_else(|| attr(app, "AXMainWindow"))
- .or_else(|| attr(app, "AXWindows").and_then(|w| list(w).into_iter().next()))
+ let window = primary_window(
+ attr(app, "AXFocusedWindow"),
+ || attr(app, "AXMainWindow"),
+ || attr(app, "AXWindows").map(list).unwrap_or_default(),
+ )?;
+ if window_id(&window) == 0 {
+ return None;
+ }
+ if bool_attr(&window, "AXMinimized") == Some(true) {
+ return Some(window);
+ }
+ let sheet = attr(&window, "AXSheets").into_iter().flat_map(list).next();
+ Some(sheet.filter(|sheet| window_id(sheet) != 0).unwrap_or(window))
}
fn window_id(w: &El) -> u32 {
let mut id = 0;
- unsafe { _AXUIElementGetWindow(w.as_CFTypeRef(), &mut id) };
- id
+ match unsafe { _AXUIElementGetWindow(w.as_CFTypeRef(), &mut id) } {
+ 0 => id,
+ _ => 0,
+ }
}
-fn line(e: &El, origin: (f64, f64)) -> (String, bool) {
- let role = str_attr(e, "AXRole").unwrap_or_else(|| "AXUnknown".into());
- let title = str_attr(e, "AXTitle");
- let desc = str_attr(e, "AXDescription").or_else(|| str_attr(e, "AXHelp"));
- let value = str_attr(e, "AXValue");
- let can_set = settable(e, "AXValue");
- let acts: Vec<String> = actions(e).into_iter().filter(|a| !BORING.contains(&a.as_str())).collect();
- let keep = title.is_some() || desc.is_some() || value.is_some() || can_set || !acts.is_empty();
- let mut s = role.trim_start_matches("AX").to_string();
- if let Some(t) = &title {
- s += &format!(" \"{}\"", clip(t, 80));
+fn exact_window(pid: i32, id: u32) -> Option<El> {
+ if id == 0 {
+ return None;
}
- if let Some(d) = desc.filter(|d| Some(d) != title.as_ref()) {
- s += &format!(" ({})", clip(&d, 80));
+ let app = app_element(pid);
+ let windows: Vec<_> = [attr(&app, "AXFocusedWindow"), attr(&app, "AXMainWindow")]
+ .into_iter()
+ .flatten()
+ .chain(attr(&app, "AXWindows").into_iter().flat_map(list))
+ .collect();
+ windows
+ .iter()
+ .cloned()
+ .chain(windows.iter().flat_map(|w| attr(w, "AXSheets").into_iter().flat_map(list)))
+ .find(|w| window_id(w) == id && bool_attr(w, "AXMinimized") != Some(true))
+}
+
+pub(crate) fn focused_window(pid: i32) -> Option<(u32, CFType)> {
+ let window = attr(&app_element(pid), "AXFocusedWindow")?;
+ let id = window_id(&window);
+ if id == 0 || unsafe { AXUIElementSetMessagingTimeout(window.as_CFTypeRef(), 0.2) } != 0 {
+ return None;
}
- if let Some(v) = value.filter(|v| Some(v) != title.as_ref()) {
- s += &format!(" = \"{}\"", clip(&v, 120));
+ Some((id, window))
+}
+
+pub(crate) fn focus_activation_window(pid: i32, id: u32) -> Option<(core_graphics::geometry::CGRect, CGPoint)> {
+ let window = exact_window(pid, id)?;
+ let f = frame(&window)?;
+ let activation: CGPoint = ax_value(&window, "AXActivationPoint", 1)?;
+ if ![f.x, f.y, f.w, f.h, activation.x, activation.y].iter().all(|n| n.is_finite()) || f.w <= 0.0 || f.h <= 0.0 {
+ return None;
}
- let mut tags = vec![];
- for (attr_name, want, tag) in [("AXFocused", true, "focused"), ("AXEnabled", false, "disabled"), ("AXSelected", true, "selected")] {
- if bool_attr(e, attr_name) == Some(want) {
- tags.push(tag);
+ Some((core_graphics::geometry::CGRect::new(&pt(f.x, f.y), &CGSize::new(f.w, f.h)), activation))
+}
+
+pub(crate) fn restore_focused_window(window: &CFType) -> Result<(), Failure> {
+ let status = unsafe { AXUIElementPerformAction(window.as_CFTypeRef(), cfs("AXRaise").as_concrete_TypeRef()) };
+ if status == 0 {
+ Ok(())
+ } else {
+ Err(fail(ErrorCode::ActionFailed, format!("Restoring original focused window failed (AXError {status})")))
+ }
+}
+
+fn on_current_space(win: u32) -> bool {
+ let Some(windows) = copy_window_info(kCGWindowListOptionOnScreenOnly, 0) else {
+ return false;
+ };
+ windows.iter().any(|item| {
+ let info = unsafe { CFDictionary::<CFString, CFType>::wrap_under_get_rule(*item as _) };
+ let key = unsafe { CFString::wrap_under_get_rule(kCGWindowNumber) };
+ info.find(key).and_then(|v| v.downcast::<CFNumber>()).and_then(|n| n.to_i64()) == Some(i64::from(win))
+ })
+}
+
+struct Node {
+ props: Props,
+ has_value: bool,
+ raw: Option<String>,
+ rendered: Option<text::RenderedText>,
+ frame: Option<Rect>,
+ children: Vec<El>,
+}
+
+thread_local! {
+ static FETCH_NAMES: CFArray<CFString> = CFArray::from_CFTypes(&FETCH.iter().map(|n| cfs(n)).collect::<Vec<_>>());
+ static READINESS_NAMES: CFArray<CFString> = CFArray::from_CFTypes(&[cfs("AXRole"), cfs("AXChildren")]);
+}
+
+fn fetch_attributes(e: &El, names: &CFArray<CFString>) -> Vec<Option<CFType>> {
+ let mut v: CFTypeRef = std::ptr::null();
+ let err = unsafe { AXUIElementCopyMultipleAttributeValues(e.as_CFTypeRef(), names.as_CFTypeRef(), 0, &mut v) };
+ let Some(values) = ax_result(err, v) else {
+ return names.iter().map(|name| attr(e, &name.to_string())).collect();
+ };
+ let values = list(values);
+ let error = |x: &CFType| unsafe { CFGetTypeID(x.as_CFTypeRef()) == AXValueGetTypeID() && AXValueGetType(x.as_CFTypeRef()) == 5 };
+ let mut out: Vec<Option<CFType>> = values.into_iter().map(|x| (!error(&x)).then_some(x)).collect();
+ out.resize(names.len() as usize, None);
+ out
+}
+
+fn point_of<T: Default>(v: &CFType, kind: u32) -> Option<T> {
+ let mut out = T::default();
+ unsafe { AXValueGetValue(v.as_CFTypeRef(), kind, &mut out as *mut T as *mut c_void) }.then_some(out)
+}
+
+fn fetch(e: &El) -> Node {
+ let v = FETCH_NAMES.with(|names| fetch_attributes(e, names));
+ let get = |name: &str| v[FETCH.iter().position(|n| *n == name).unwrap()].as_ref();
+ let s = |name: &str| get(name).and_then(text_of).map(|s| s.trim().to_string()).filter(|s| !s.is_empty());
+ let b = |name: &str| get(name).and_then(|v| v.downcast::<CFBoolean>()).map(bool::from);
+ let title = s("AXTitle");
+ let label = if title.is_none() {
+ get("AXTitleUIElement").and_then(|l| str_attr(l, "AXValue").or_else(|| str_attr(l, "AXTitle")))
+ } else {
+ None
+ };
+ let has_value = get("AXValue").is_some();
+ let role = s("AXRole").unwrap_or_else(|| "AXUnknown".into());
+ let visible_text = matches!(role.as_str(), "AXTextArea" | "AXTextField")
+ .then(|| {
+ let value = get("AXValue").and_then(text_of)?;
+ let range = get("AXVisibleCharacterRange").and_then(|v| point_of::<[isize; 2]>(v, 4))?;
+ crate::text::visible_text(&value, range, tree::MAX_TEXT)
+ })
+ .flatten();
+ let props = Props {
+ role,
+ visible_text,
+ rich_text: None,
+ subrole: s("AXSubrole"),
+ title,
+ description: s("AXDescription").or_else(|| s("AXHelp")),
+ label,
+ value: s("AXValue"),
+ placeholder: s("AXPlaceholderValue"),
+ url: s("AXURL"),
+ identifier: s("AXIdentifier"),
+ expanded: b("AXExpanded"),
+ focused: b("AXFocused") == Some(true),
+ disabled: b("AXEnabled") == Some(false),
+ selected: b("AXSelected") == Some(true),
+ settable: has_value,
+ actions: vec![],
+ list_item: false,
+ };
+ let frame = match (get("AXPosition").and_then(|p| point_of::<CGPoint>(p, 1)), get("AXSize").and_then(|p| point_of::<CGSize>(p, 2))) {
+ (Some(p), Some(z)) => Some(Rect { x: p.x, y: p.y, w: z.width, h: z.height }),
+ _ => None,
+ };
+ let children = get("AXChildren").cloned().map(list).unwrap_or_default();
+ let raw = get("AXValue").or(get("AXTitle")).and_then(text_of);
+ Node { props, has_value, raw, rendered: None, frame, children }
+}
+
+impl Node {
+ fn marker_candidate(&self, clip: Rect) -> bool {
+ !self.has_value
+ && self.children.is_empty()
+ && matches!(self.props.role.as_str(), "AXStaticText" | "AXTextArea" | "AXTextField")
+ && self.frame.is_some_and(|frame| frame.w > 0.0 && frame.h > 0.0 && tree::intersect(frame, clip) == Some(frame))
+ }
+
+ fn limit_text(&mut self, budget: &mut usize) {
+ if let Some(rendered) = &self.rendered {
+ if let Some(remaining) = budget.checked_sub(rendered.retained_bytes()) {
+ *budget = remaining;
+ } else {
+ self.rendered = None;
+ self.props.rich_text = None;
+ }
}
}
- if can_set {
- tags.push("settable");
+}
+
+fn rebind_text(text: HashMap<u32, text::ObservedText>, moved: &HashMap<u32, u32>) -> HashMap<u32, text::ObservedText> {
+ text.into_iter().map(|(id, text)| (*moved.get(&id).unwrap_or(&id), text)).collect()
+}
+
+fn attributed_supported(e: &El) -> bool {
+ let mut names = std::ptr::null();
+ let status = unsafe { AXUIElementCopyParameterizedAttributeNames(e.as_CFTypeRef(), &mut names) };
+ ax_result(status, names).map(list).unwrap_or_default().iter().any(|name| text_of(name).as_deref() == Some("AXAttributedStringForRange"))
+}
+
+fn native_font_traits(name: &str, size: f64) -> Option<NSFontDescriptorSymbolicTraits> {
+ if !size.is_finite() || size <= 0.0 {
+ return None;
}
- if !tags.is_empty() {
- s += &format!(" [{}]", tags.join(" "));
+ let system = NSFont::systemFontOfSize(size).fontDescriptor();
+ for traits in [
+ NSFontDescriptorSymbolicTraits::empty(),
+ NSFontDescriptorSymbolicTraits::TraitBold,
+ NSFontDescriptorSymbolicTraits::TraitItalic,
+ NSFontDescriptorSymbolicTraits::TraitBold | NSFontDescriptorSymbolicTraits::TraitItalic,
+ ] {
+ let descriptor = system.fontDescriptorWithSymbolicTraits(traits);
+ if let Some(font) = NSFont::fontWithDescriptor_size(&descriptor, size).filter(|font| font.fontName().to_string() == name) {
+ return Some(font.fontDescriptor().symbolicTraits());
+ }
}
- if !acts.is_empty() {
- let a: Vec<&str> = acts.iter().map(|a| a.trim_start_matches("AX")).collect();
- s += &format!(" <{}>", a.join(","));
+ NSFont::fontWithName_size(&NSString::from_str(name), size)
+ .filter(|font| font.fontName().to_string() == name)
+ .map(|font| font.fontDescriptor().symbolicTraits())
+}
+
+fn attributed_runs(value: &CFAttributedString, expected: &str) -> Option<Vec<(String, text::TextStyle)>> {
+ let plain = unsafe { CFString::wrap_under_get_rule(CFAttributedStringGetString(value.as_concrete_TypeRef())) }.to_string();
+ if plain != expected {
+ return None;
}
- if let Some(f) = frame(e) {
- s += &format!(" @{:.0},{:.0} {:.0}x{:.0}", f.x - origin.0, f.y - origin.1, f.w, f.h);
+ let mut runs = vec![];
+ let mut fonts = HashMap::new();
+ let mut at = 0;
+ while at < value.char_len() {
+ if cancellation::cancelled() {
+ return None;
+ }
+ let mut range = CFRange { location: 0, length: 0 };
+ let attrs = unsafe { CFAttributedStringGetAttributes(value.as_concrete_TypeRef(), at, &mut range) };
+ if attrs.is_null() || range.location > at || range.length <= 0 {
+ return None;
+ }
+ let end = range.location.checked_add(range.length)?;
+ if end <= at || end > value.char_len() {
+ return None;
+ }
+ let attrs = unsafe { CFDictionary::<CFString, CFType>::wrap_under_get_rule(attrs) };
+ let get = |key: &str| attrs.find(cfs(key)).map(|v| v.clone());
+ let number = |key: &str| get(key).and_then(|v| v.downcast::<CFNumber>()).and_then(|v| v.to_i64());
+ let flag = |key: &str| {
+ get(key).and_then(|v| v.downcast::<CFBoolean>()).map(bool::from).unwrap_or_else(|| number(key).is_some_and(|v| v != 0))
+ };
+ let font_traits = get("AXFont")
+ .and_then(|font| {
+ let font = font.downcast::<CFDictionary>()?;
+ let font = unsafe { CFDictionary::<CFString, CFType>::wrap_under_get_rule(font.as_concrete_TypeRef()) };
+ let name = font.find(cfs("AXFontName")).and_then(|value| text_of(&value))?;
+ let size = font.find(cfs("AXFontSize")).and_then(|value| value.downcast::<CFNumber>()).and_then(|value| value.to_f64())?;
+ *fonts.entry((name.clone(), size.to_bits())).or_insert_with(|| native_font_traits(&name, size))
+ })
+ .unwrap_or_else(NSFontDescriptorSymbolicTraits::empty);
+ let list_prefix = get("AXListItemPrefix")
+ .and_then(|v| v.downcast::<CFAttributedString>())
+ .map(|v| unsafe { CFString::wrap_under_get_rule(CFAttributedStringGetString(v.as_concrete_TypeRef())) }.to_string());
+ let link = if let Some(link) = get("AXLink").filter(|link| link.type_of() == unsafe { AXUIElementGetTypeID() }) {
+ if cancellation::cancelled() {
+ return None;
+ }
+ str_attr(&link, "AXURL").filter(|url| url.len() <= 2048)
+ } else {
+ None
+ };
+ let style = text::TextStyle {
+ bold: get("AXFontBold").map_or_else(|| font_traits.contains(NSFontDescriptorSymbolicTraits::TraitBold), |_| flag("AXFontBold")),
+ italic: get("AXFontItalic")
+ .map_or_else(|| font_traits.contains(NSFontDescriptorSymbolicTraits::TraitItalic), |_| flag("AXFontItalic")),
+ underline: flag("AXUnderline"),
+ strike: flag("AXStrikethrough"),
+ link,
+ list_prefix,
+ list_level: number("AXListItemLevel").and_then(|v| usize::try_from(v).ok()).unwrap_or(0),
+ };
+ let run = text::visible_text(&plain, [at, end - at], tree::MAX_TEXT)?;
+ runs.push((run.text, style));
+ at = end;
+ }
+ Some(runs)
+}
+
+fn attributed_text(e: &El, n: &Node) -> Option<text::RenderedText> {
+ let raw = n.raw.as_deref()?;
+ let prefix;
+ let visible = match n.props.visible_text.as_ref() {
+ Some(visible) => visible,
+ None => {
+ prefix = text::visible_text(raw, [0, isize::try_from(raw.encode_utf16().count()).ok()?], tree::MAX_TEXT)?;
+ &prefix
+ }
+ };
+ if visible.text.is_empty() {
+ return None;
+ }
+ let range = [isize::try_from(visible.start).ok()?, isize::try_from(visible.shown_end - visible.start).ok()?];
+ let parameter = unsafe { AXValueCreate(4, range.as_ptr() as *const c_void) };
+ if parameter.is_null() {
+ return None;
}
- (s, keep)
+ let parameter = unsafe { CFType::wrap_under_create_rule(parameter) };
+ let mut value = std::ptr::null();
+ let result = unsafe {
+ AXUIElementCopyParameterizedAttributeValue(
+ e.as_CFTypeRef(),
+ cfs("AXAttributedStringForRange").as_concrete_TypeRef(),
+ parameter.as_CFTypeRef(),
+ &mut value,
+ )
+ };
+ let value = ax_result(result, value)?.downcast::<CFAttributedString>()?;
+ let runs = attributed_runs(&value, &visible.text)?;
+ let starts_line = visible.start == 0 || raw.encode_utf16().nth(visible.start - 1) == Some(10);
+ let mut markdown = text::attributed_text(&runs, starts_line)?;
+ markdown.offset(visible.start);
+ if visible.shown_end < visible.end {
+ markdown.syntax("\n… text truncated");
+ }
+ Some(markdown)
+}
+
+fn complete(e: &El, mut n: Node, cache: Option<&mut Cache>) -> Node {
+ let hash = unsafe { CFHash(e.as_CFTypeRef()) };
+ let text_role = matches!(n.props.role.as_str(), "AXTextArea" | "AXTextField" | "AXStaticText");
+ let rich_candidate = text_role && (n.props.role != "AXStaticText" || n.children.is_empty());
+ let cached = cache.as_ref().and_then(|c| c.get(e, hash));
+ let uncached = cached.is_none();
+ let metadata = cached.unwrap_or_else(|| Metadata {
+ actions: actions(e).into_iter().filter(|a| !BORING.contains(&a.as_str())).collect(),
+ settable: n.props.settable && settable(e, "AXValue"),
+ attributed: text_role && attributed_supported(e),
+ });
+ if metadata.attributed && rich_candidate {
+ n.rendered = attributed_text(e, &n);
+ n.props.rich_text = n.rendered.as_ref().map(|rendered| rendered.markdown.clone());
+ }
+ n.props.actions = metadata.actions.clone();
+ n.props.settable = metadata.settable;
+ if let Some(c) = cache {
+ if uncached {
+ c.put(e, hash, metadata);
+ }
+ }
+ n
+}
+
+fn extras(ax: &El, main: &El) -> Vec<El> {
+ let mut out: Vec<El> = attr(ax, "AXChildren")
+ .map(list)
+ .unwrap_or_default()
+ .into_iter()
+ .filter(|c| str_attr(c, "AXRole").as_deref() == Some("AXMenu"))
+ .collect();
+ let bar = attr(ax, "AXMenuBar").map(|b| attr(&b, "AXChildren").map(list).unwrap_or_default()).unwrap_or_default();
+ for item in bar.iter().filter(|i| bool_attr(i, "AXSelected") == Some(true)) {
+ out.extend(
+ attr(item, "AXChildren")
+ .map(list)
+ .unwrap_or_default()
+ .into_iter()
+ .filter(|c| str_attr(c, "AXRole").as_deref() == Some("AXMenu")),
+ );
+ }
+ for w in attr(ax, "AXWindows").map(list).unwrap_or_default() {
+ let role = str_attr(&w, "AXRole").unwrap_or_default();
+ let sub = str_attr(&w, "AXSubrole").unwrap_or_default();
+ if &w != main && (POPUPS.contains(&role.as_str()) || TRANSIENT.contains(&sub.as_str())) && !out.contains(&w) {
+ out.push(w);
+ }
+ }
+ out
+}
+
+#[derive(Clone, Copy)]
+struct WalkAt {
+ depth: usize,
+ anchor: u32,
+ clip: Rect,
+ at: Locator,
}
struct Walk {
- origin: (f64, f64),
- known: HashMap<usize, Vec<(u32, El)>>,
- taken: HashSet<u32>,
- next: u32,
+ ids: tree::Ids<El>,
elements: Vec<(u32, El)>,
- lines: Vec<(u32, String)>,
+ popups: Vec<El>,
+ text: HashMap<u32, text::ObservedText>,
+ markers: HashMap<u32, (El, String)>,
+ ancestry: HashMap<u32, Vec<El>>,
+ text_budget: usize,
+ notes: HashMap<u32, u32>,
truncated: bool,
visited: usize,
+ ancestors: Vec<El>,
+ window: Rect,
+ cache: Option<Cache>,
}
impl Walk {
- fn id(&mut self, e: &El) -> u32 {
- let found = self
- .known
- .get(&unsafe { CFHash(e.as_CFTypeRef()) })
- .and_then(|c| c.iter().find(|(id, k)| !self.taken.contains(id) && k == e))
- .map(|(id, _)| *id);
- let id = found.unwrap_or_else(|| {
- self.next += 1;
- self.next - 1
- });
- self.taken.insert(id);
+ fn id(&mut self, e: &El, locator: Locator) -> u32 {
+ let id = self.ids.assign(e, unsafe { CFHash(e.as_CFTypeRef()) }, locator);
+ self.elements.push((id, e.clone()));
+ let mut path = self.ancestors.clone();
+ path.push(e.clone());
+ self.ancestry.insert(id, path);
id
}
- fn visit(&mut self, e: &El, depth: usize, indent: usize) {
- if self.visited >= MAX_NODES || depth > MAX_DEPTH {
+ fn full(&self) -> bool {
+ self.visited >= MAX_VISITS || cancellation::cancelled()
+ }
+
+ fn note(&mut self, anchor: u32, s: String) -> Item {
+ let seq = self.notes.entry(anchor).or_default();
+ let key = Key::Note(anchor, *seq);
+ *seq += 1;
+ Item::leaf(key, s)
+ }
+
+ fn fold(&mut self, n: &mut Node) -> Option<Vec<Node>> {
+ if !n.props.folds_text() || n.children.is_empty() || n.children.len() > 20 || self.visited + n.children.len() > MAX_VISITS {
+ return None;
+ }
+ self.visited += n.children.len();
+ let kids: Vec<Node> = n.children.iter().map(|c| complete(c, fetch(c), self.cache.as_mut())).collect();
+ if kids.iter().all(|k| k.has_value && k.props.plain_text() && k.children.is_empty()) {
+ let parts: Vec<(String, Option<Rect>)> = kids.iter().filter_map(|k| Some((k.raw.clone()?, k.frame))).collect();
+ n.props.title = Some(tree::joined_text(&parts)).filter(|t| !t.is_empty());
+ n.children.clear();
+ None
+ } else {
+ Some(kids)
+ }
+ }
+
+ fn visit(&mut self, e: &El, mut n: Node, context: WalkAt, force: bool, out: &mut Vec<Item>) {
+ let WalkAt { depth, anchor, clip, at } = context;
+ if depth > MAX_DEPTH {
self.truncated = true;
return;
}
- self.visited += 1;
- let (s, keep) = line(e, self.origin);
- let next = if keep {
- let id = self.id(e);
- self.lines.push((id, format!("{}[{}] {}", " ".repeat(indent), id, s)));
- self.elements.push((id, e.clone()));
- indent + 1
+ let hash = unsafe { CFHash(e.as_CFTypeRef()) };
+ if self.ids.seen(e, hash) {
+ return;
+ }
+ if n.props.role == "AXPopUpButton" {
+ self.popups.push(e.clone());
+ }
+ let locator = at.child(&n.props);
+ let mut marker_source = None;
+ let mut marker_note = None;
+ if n.marker_candidate(clip) {
+ match markers::read(e) {
+ Some(markers::Read::Text { range, raw, attributed }) => {
+ let retained = raw.capacity() * 3 + std::mem::size_of::<(El, String)>();
+ if let Some(remaining) = self.text_budget.checked_sub(retained) {
+ self.text_budget = remaining;
+ n.rendered = attributed_runs(&attributed, &raw).and_then(|runs| text::attributed_text(&runs, false));
+ n.props.rich_text = n.rendered.as_ref().map(|rendered| rendered.markdown.clone());
+ n.props.value = Some(raw.clone());
+ n.raw = Some(raw.clone());
+ marker_note = Some(format!(
+ "Text from element marker range: {} UTF-16 units; text selection unavailable",
+ raw.encode_utf16().count()
+ ));
+ marker_source = Some((range, raw));
+ } else {
+ self.truncated = true;
+ marker_note = Some("… marker text omitted; observation text budget reached".into());
+ }
+ }
+ Some(markers::Read::Omitted(length)) => {
+ self.truncated = true;
+ marker_note = Some(format!(
+ "… element marker range length is {length}; text not read because it exceeds the {}-unit limit",
+ tree::MAX_TEXT
+ ));
+ }
+ None => {}
+ }
+ }
+ let prefetched = self.fold(&mut n);
+ n.limit_text(&mut self.text_budget);
+ let (s, keep) = n.props.describe();
+ let keep = keep || force || marker_note.is_some();
+ let clip = narrow(clip, &n.props.role, n.frame);
+ let drawn = n.props.role == "AXWebArea" || n.children.is_empty();
+ let (anchor, mut kids) = if keep { (self.id(e, locator), vec![]) } else { (anchor, vec![]) };
+ self.ancestors.push(e.clone());
+ self.children(&n.children, prefetched, WalkAt { depth: depth + 1, anchor, clip, at: locator }, n.props.content_list(), &mut kids);
+ self.ancestors.pop();
+ if kids.is_empty()
+ && drawn
+ && !self.truncated
+ && SPARSE.contains(&n.props.role.as_str())
+ && n.frame.is_some_and(|f| tree::sparse(f, self.window, 0))
+ {
+ let note =
+ "… nothing readable inside; the app may draw this area itself (for example a canvas). Use the screenshot and point targets";
+ kids.push(self.note(anchor, note.into()));
+ }
+ if keep {
+ if let Some(note) = marker_note {
+ kids.push(self.note(anchor, note));
+ }
+ if let Some(source) = marker_source {
+ self.markers.insert(anchor, source);
+ } else if let (Some(value), Some(rendered)) = (n.raw, n.rendered) {
+ self.text.insert(anchor, text::ObservedText::new(&value, rendered));
+ }
+ out.push(Item { key: Key::Element(anchor), text: s, kids });
} else {
- indent
- };
- for c in attr(e, "AXChildren").map(list).unwrap_or_default() {
- if self.visited >= MAX_NODES {
+ out.extend(kids);
+ }
+ }
+
+ fn text_run(&mut self, run: &mut Vec<(El, Node)>, context: WalkAt, out: &mut Vec<Item>) {
+ match run.len() {
+ 0 => {}
+ 1 => {
+ let (e, n) = run.pop().unwrap();
+ self.visit(&e, n, context, false, out);
+ }
+ _ => {
+ let texts: Vec<(String, Option<Rect>)> = run.iter().filter_map(|(_, n)| Some((n.raw.clone()?, n.frame))).collect();
+ let (first, n) = &run[0];
+ let id = self.id(first, context.at.child(&n.props));
+ out.push(Item::leaf(Key::Element(id), tree::merged_text(&texts)));
+ }
+ }
+ run.clear();
+ }
+
+ fn children(&mut self, kids: &[El], prefetched: Option<Vec<Node>>, context: WalkAt, list: bool, out: &mut Vec<Item>) {
+ let WalkAt { anchor, clip, .. } = context;
+ let mut prefetched = prefetched.unwrap_or_default().into_iter();
+ let mut hidden = 0;
+ let mut run: Vec<(El, Node)> = vec![];
+ for (i, c) in kids.iter().enumerate() {
+ let prefetched_node = prefetched.next();
+ if cancellation::cancelled() || (prefetched_node.is_none() && self.full()) {
self.truncated = true;
+ self.text_run(&mut run, context, out);
+ let note = format!("… {} more not shown; the observation size limit was reached", kids.len() - i);
+ out.push(self.note(anchor, note));
break;
}
- self.visit(&c, depth + 1, next);
+ let loaded = prefetched_node.is_some();
+ let n = prefetched_node.unwrap_or_else(|| {
+ self.visited += 1;
+ fetch(c)
+ });
+ if n.frame.is_some_and(|f| tree::visible(f, clip).is_none()) {
+ hidden += 1;
+ continue;
+ }
+ let mut n = if loaded { n } else { complete(c, n, self.cache.as_mut()) };
+ n.props.list_item = list;
+ if n.props.plain_text() && n.has_value && n.children.is_empty() {
+ run.push((c.clone(), n));
+ continue;
+ }
+ self.text_run(&mut run, context, out);
+ self.visit(c, n, context, false, out);
+ }
+ self.text_run(&mut run, context, out);
+ if hidden > 0 {
+ out.push(self.note(anchor, format!("… {hidden} more offscreen; scroll to reveal")));
+ }
+ }
+}
+
+fn relocate(roots: &[(El, Locator)], target: Locator) -> Option<El> {
+ let mut queue: std::collections::VecDeque<(El, Locator)> = roots.iter().cloned().collect();
+ let (mut found, mut seen) = (None, 0);
+ while let Some((e, at)) = queue.pop_front() {
+ seen += 1;
+ if seen > MAX_VISITS || cancellation::cancelled() {
+ return None;
+ }
+ let n = fetch(&e);
+ let here = at.child(&n.props);
+ if here == target {
+ if found.is_some() {
+ return None;
+ }
+ found = Some(e);
+ }
+ queue.extend(n.children.into_iter().map(|c| (c, here)));
+ }
+ found
+}
+
+fn web_ready<T>(root: T, required: bool, mut read: impl FnMut(&T) -> Option<(bool, Vec<T>)>) -> Option<bool> {
+ let mut stack = vec![(root, 0)];
+ let mut seen = 0;
+ let mut found = false;
+ while let Some((element, depth)) = stack.pop() {
+ seen += 1;
+ if seen > MAX_BUSY_VISITS || depth > MAX_DEPTH {
+ return None;
+ }
+ let (web, children) = read(&element)?;
+ if web {
+ found = true;
+ if children.is_empty() {
+ return Some(false);
+ }
+ } else {
+ stack.extend(children.into_iter().map(|child| (child, depth + 1)));
}
}
+ Some(found || !required)
+}
+
+fn electron_app(app: &App) -> bool {
+ app.path
+ .as_ref()
+ .and_then(|path| path.to_str())
+ .and_then(|path| NSBundle::bundleWithPath(&NSString::from_str(path)))
+ .and_then(|bundle| bundle.objectForInfoDictionaryKey(&NSString::from_str("NSPrincipalClass")))
+ .and_then(|value| value.downcast::<NSString>().ok())
+ .is_some_and(|class| class.to_string() == "AtomApplication")
+}
+
+fn narrow(clip: Rect, role: &str, f: Option<Rect>) -> Rect {
+ match f {
+ Some(f) if CLIPS.contains(&role) => tree::intersect(f, clip).unwrap_or(Rect { w: 0.0, h: 0.0, ..clip }),
+ _ => clip,
+ }
+}
+
+fn visible_rect(e: &El, f: Rect, window: Rect) -> Option<Rect> {
+ let mut clips = vec![];
+ let mut root = None;
+ let mut parent = attr(e, "AXParent");
+ for _ in 0..MAX_DEPTH {
+ let Some(p) = parent else { break };
+ let role = str_attr(&p, "AXRole").unwrap_or_default();
+ if role == "AXWindow" || role == "AXApplication" {
+ root = Some((role, p));
+ break;
+ }
+ if CLIPS.contains(&role.as_str()) {
+ clips.push((role, frame(&p)));
+ }
+ parent = attr(&p, "AXParent");
+ }
+ let everywhere = Rect { x: -1e9, y: -1e9, w: 2e9, h: 2e9 };
+ let base = match root {
+ Some((role, w)) if role == "AXWindow" => frame(&w).unwrap_or(window),
+ Some(_) => everywhere,
+ None => window,
+ };
+ let clip = clips.iter().rev().fold(base, |c, (role, f)| narrow(c, role, *f));
+ tree::visible(f, clip)
}
fn busy(e: &El, depth: usize, seen: &mut usize) -> bool {
*seen += 1;
- if *seen > MAX_NODES || depth > MAX_DEPTH {
+ if *seen > MAX_BUSY_VISITS || depth > MAX_DEPTH {
return false;
}
- if str_attr(e, "AXRole").is_some_and(|r| BUSY.contains(&r.as_str())) {
+ let mut values = READINESS_NAMES.with(|names| fetch_attributes(e, names)).into_iter();
+ if values.next().flatten().as_ref().and_then(text_of).is_some_and(|role| BUSY.contains(&role.as_str())) {
return true;
}
- attr(e, "AXChildren").map(list).unwrap_or_default().iter().any(|c| busy(c, depth + 1, seen))
-}
-
-fn select(e: &El, text: &str, prefix: &str, suffix: &str, selection: Selection) -> Result<(), Failure> {
- let value = attr(e, "AXValue").and_then(|v| text_of(&v)).unwrap_or_default();
- let at = value
- .match_indices(text)
- .map(|(i, _)| i)
- .find(|&i| value[..i].ends_with(prefix) && value[i + text.len()..].starts_with(suffix))
- .ok_or_else(|| fail(ErrorCode::ActionFailed, format!("text {text:?} not found in the element's value")))?;
- let start = value[..at].encode_utf16().count() as isize;
- let len = text.encode_utf16().count() as isize;
- let range: [isize; 2] = match selection {
- Selection::Text => [start, len],
- Selection::CursorBefore => [start, 0],
- Selection::CursorAfter => [start + len, 0],
- };
- set_attr(e, "AXFocused", &CFBoolean::true_value().as_CFType());
+ values.next().flatten().map(list).unwrap_or_default().iter().any(|c| busy(c, depth + 1, seen))
+}
+
+fn select(
+ focus: impl FnOnce() -> Result<(), Failure>,
+ e: &El,
+ observed: Option<&text::ObservedText>,
+ text: &str,
+ prefix: &str,
+ suffix: &str,
+ selection: Selection,
+) -> Result<(), Failure> {
+ if !settable(e, "AXSelectedTextRange") {
+ return Err(fail(ErrorCode::ActionFailed, "The element does not support setting its text selection"));
+ }
+ let value = attr(e, "AXValue")
+ .and_then(|v| text_of(&v))
+ .ok_or_else(|| fail(ErrorCode::ActionFailed, "The element's text value could not be read"))?;
+ let range = text::observed_selection_range(&value, observed, text, prefix, suffix, selection)
+ .map_err(|message| fail(ErrorCode::ActionFailed, message))?;
+ focus()?;
+ if attr(e, "AXValue").and_then(|v| text_of(&v)).as_ref() != Some(&value) {
+ return Err(fail(ErrorCode::ActionFailed, "The element's text changed while focusing it; observe it again before selecting"));
+ }
+ active()?;
let v = unsafe { CFType::wrap_under_create_rule(AXValueCreate(4, range.as_ptr() as *const c_void)) };
match set_attr(e, "AXSelectedTextRange", &v) {
0 => Ok(()),
@@ -485,28 +1467,33 @@ fn select(e: &El, text: &str, prefix: &str, suffix: &str, selection: Selection)
}
}
-fn png(img: &core_graphics::image::CGImage) -> Option<(String, u32, u32)> {
- let (w, h, row) = (img.width(), img.height(), img.bytes_per_row());
- if img.bits_per_pixel() != 32 {
- return None;
- }
- let data = img.data();
- let bytes = data.bytes();
- let mut rgba = Vec::with_capacity(w * h * 4);
- for y in 0..h {
- for x in 0..w {
- let i = y * row + x * 4;
- rgba.extend_from_slice(&[bytes[i + 2], bytes[i + 1], bytes[i], bytes[i + 3]]);
+fn quiet(since: Instant, notices: &Notices, minimum: Duration, silence: Duration, maximum: Duration) {
+ let start = now_ms() - since.elapsed().as_millis() as i64;
+ loop {
+ CFRunLoop::run_in_mode(unsafe { kCFRunLoopDefaultMode }, Duration::from_millis(50), false);
+ let elapsed = since.elapsed();
+ let last = notices.last.load(Ordering::Relaxed).max(start);
+ if active().is_err() || elapsed >= maximum {
+ return;
+ }
+ if elapsed >= minimum && now_ms() - last >= silence.as_millis() as i64 {
+ return;
}
}
- let mut buf = Vec::new();
- {
- let mut enc = png::Encoder::new(&mut buf, w as u32, h as u32);
- enc.set_color(png::ColorType::Rgba);
- enc.set_depth(png::BitDepth::Eight);
- enc.write_header().ok()?.write_image_data(&rgba).ok()?;
+}
+
+fn settle(w: &El, since: Instant, notices: Option<&Notices>) {
+ if let Some(n) = notices {
+ quiet(since, n, SETTLE_MIN, SETTLE_QUIET, SETTLE_MAX);
+ } else {
+ cancellation::wait(SETTLE.saturating_sub(since.elapsed()));
+ }
+ for _ in 0..16 {
+ if cancellation::cancelled() || !busy(w, 0, &mut 0) {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(300));
}
- Some((base64::engine::general_purpose::STANDARD.encode(buf), w as u32, h as u32))
}
fn permissions(prompt: bool) -> Reply {
@@ -516,13 +1503,7 @@ fn permissions(prompt: bool) -> Reply {
} else {
unsafe { AXIsProcessTrusted() }
};
- let screen_recording = unsafe {
- if prompt {
- CGRequestScreenCaptureAccess()
- } else {
- CGPreflightScreenCaptureAccess()
- }
- };
+ let screen_recording = if prompt { CGRequestScreenCaptureAccess() } else { CGPreflightScreenCaptureAccess() };
let binary = std::env::current_exe().map(|p| p.display().to_string()).unwrap_or_default();
let help = "System Settings > Privacy & Security > Accessibility and > Screen Recording: add the app that launches desk \
(QM, or your terminal when run alone) with +, switch it on, then quit and reopen that app. \
@@ -532,7 +1513,23 @@ fn permissions(prompt: bool) -> Reply {
}
impl Desk {
+ fn sync_app(&mut self, app: &App) {
+ if self.launches.get(&app.pid) == Some(&app.launched) {
+ return;
+ }
+ let _ = self.enforcers.remove(app.pid, false);
+ self.cache.remove(&app.pid);
+ self.touched.remove(&app.pid);
+ self.enabled.remove(&app.pid);
+ self.observers.remove(&app.pid);
+ self.launches.insert(app.pid, app.launched);
+ }
+
pub fn handle(&mut self, cmd: Command, pace: &mut dyn FnMut(u32, Point) -> Result<(), Failure>) -> Reply {
+ objc2::rc::autoreleasepool(|_| self.handle_inner(cmd, pace))
+ }
+
+ fn handle_inner(&mut self, cmd: Command, pace: &mut dyn FnMut(u32, Point) -> Result<(), Failure>) -> Reply {
let r = match cmd {
Command::Permissions { prompt } => Ok(permissions(prompt)),
Command::Apps => apps(),
@@ -548,84 +1545,180 @@ impl Desk {
r.unwrap_or_else(Reply::Error)
}
+ fn enable(&mut self, app: &App, ax: &El, w: &El) -> Result<(), Failure> {
+ if self.enabled.get(&app.pid).is_some_and(|state| state.ready) {
+ return Ok(());
+ }
+ if str_attr(ax, "AXRole").as_deref() != Some("AXApplication") {
+ return Err(fail(ErrorCode::ActionFailed, "Accessibility application root is unavailable; observe again"));
+ }
+ let required = electron_app(app);
+ if let std::collections::hash_map::Entry::Vacant(slot) = self.enabled.entry(app.pid) {
+ let mut prior = vec![];
+ let mut turn_on = |name: &'static str| {
+ let was = bool_attr(ax, name);
+ let on = was == Some(true) || set_attr(ax, name, &CFBoolean::true_value().as_CFType()) == 0;
+ if on && was != Some(true) {
+ prior.push((name, false));
+ }
+ on
+ };
+ if !turn_on("AXManualAccessibility") {
+ turn_on("AXEnhancedUserInterface");
+ }
+ slot.insert(Enablement { prior, ready: false });
+ }
+ if required && str_attr(w, "AXRole").as_deref() == Some("AXSheet") {
+ return Ok(());
+ }
+ for attempt in 0..=10 {
+ if cancellation::cancelled() {
+ return Err(fail(ErrorCode::ActionFailed, "Accessibility enablement cancelled"));
+ }
+ let ready = web_ready(w.clone(), required, |element| {
+ if cancellation::cancelled() {
+ return None;
+ }
+ let mut values = READINESS_NAMES.with(|names| fetch_attributes(element, names)).into_iter();
+ let role = values.next().flatten().as_ref().and_then(text_of)?;
+ let children = values.next().flatten().map(list).unwrap_or_default();
+ Some((role == "AXWebArea", children))
+ });
+ if cancellation::cancelled() {
+ return Err(fail(ErrorCode::ActionFailed, "Accessibility enablement cancelled"));
+ }
+ if ready == Some(true) || (!required && ready.is_none()) {
+ self.enabled.get_mut(&app.pid).unwrap().ready = true;
+ return Ok(());
+ }
+ if attempt < 10 {
+ cancellation::wait(Duration::from_millis(100));
+ }
+ }
+ if required {
+ Err(fail(ErrorCode::ActionFailed, "Electron accessibility content is not ready; observe again"))
+ } else {
+ self.enabled.get_mut(&app.pid).unwrap().ready = true;
+ Ok(())
+ }
+ }
+
fn observe(&mut self, app: &App, shot: bool, diff: bool) -> Result<Observation, Failure> {
- let (_, w) = window(app)?;
+ self.sync_app(app);
+ if let Some(at) = self.touched.remove(&app.pid) {
+ let (_, w) = window(app)?;
+ settle(&w, at, self.observers.get(&app.pid).map(|o| &*o.notices));
+ }
+ let (ax, w) = window(app)?;
+ self.enable(app, &ax, &w)?;
+ if let std::collections::hash_map::Entry::Vacant(slot) = self.observers.entry(app.pid) {
+ if let Some(o) = Observer::new(app.pid, &ax) {
+ slot.insert(o);
+ }
+ }
let win = window_id(&w);
let f = frame(&w).unwrap_or(Rect { x: 0.0, y: 0.0, w: 0.0, h: 0.0 });
- let prev = self.cache.remove(&app.pid).filter(|s| s.window == win);
+ let mut prev = self.cache.remove(&app.pid).filter(|s| s.window == win);
let mut walk = Walk {
- origin: (f.x, f.y),
- known: HashMap::new(),
- taken: HashSet::new(),
- next: prev.as_ref().map_or(0, |p| p.next),
+ ids: prev.as_mut().map(|p| std::mem::take(&mut p.ids)).unwrap_or_default(),
elements: vec![],
- lines: vec![],
+ popups: vec![],
+ text: HashMap::new(),
+ markers: HashMap::new(),
+ ancestry: HashMap::new(),
+ text_budget: 1024 * 1024,
+ notes: HashMap::new(),
truncated: false,
visited: 0,
+ ancestors: vec![],
+ window: f,
+ cache: self.observers.get_mut(&app.pid).map(Observer::take_cache),
};
- for (id, e) in prev.iter().flat_map(|p| p.elements.iter()) {
- walk.known.entry(unsafe { CFHash(e.as_CFTypeRef()) }).or_default().push((*id, e.clone()));
+ let mut items = vec![];
+ let everywhere = Rect { x: -1e9, y: -1e9, w: 2e9, h: 2e9 };
+ let others = extras(&ax, &w);
+ if !others.is_empty() {
+ items.push(Item::leaf(Key::Section(0), "Open menus and other windows of this app:".into()));
+ for o in &others {
+ let n = complete(o, fetch(o), walk.cache.as_mut());
+ walk.ancestors = attr(o, "AXParent").into_iter().collect();
+ walk.visit(o, n, WalkAt { depth: 0, anchor: ROOT, clip: everywhere, at: Locator::ROOT }, true, &mut items);
+ }
+ let title = str_attr(&w, "AXTitle").unwrap_or_default();
+ items.push(Item::leaf(Key::Section(1), format!("Window \"{}\":", tree::clip(&title, 80))));
}
- for c in attr(&w, "AXChildren").map(list).unwrap_or_default() {
- if walk.visited >= MAX_NODES {
- walk.truncated = true;
- break;
+ let kids = attr(&w, "AXChildren").map(list).unwrap_or_default();
+ walk.ancestors = vec![w.clone()];
+ walk.children(
+ &kids,
+ None,
+ WalkAt { depth: 0, anchor: ROOT, clip: if f.w > 0.0 && f.h > 0.0 { f } else { everywhere }, at: Locator::ROOT },
+ false,
+ &mut items,
+ );
+ if let (Some(o), Some(c)) = (self.observers.get_mut(&app.pid), walk.cache.take()) {
+ o.cache = c;
+ }
+ let moved = walk.ids.rebind(dead);
+ items.iter_mut().for_each(|i| i.renumber(&moved));
+ walk.text = rebind_text(walk.text, &moved);
+ walk.ancestry = walk.ancestry.into_iter().map(|(id, path)| (*moved.get(&id).unwrap_or(&id), path)).collect();
+ walk.markers = walk.markers.into_iter().map(|(id, source)| (*moved.get(&id).unwrap_or(&id), source)).collect();
+ for (id, _) in walk.elements.iter_mut() {
+ *id = *moved.get(id).unwrap_or(id);
+ }
+ let (mut lines, cut) = tree::layout(&items, MAX_TREE_CHARS);
+ if let Some(focus) = attr(&ax, "AXFocusedUIElement") {
+ let what = match walk.elements.iter().find(|(_, e)| *e == focus) {
+ Some((id, _)) if lines.iter().any(|(k, _)| *k == Key::Element(*id)) => format!("[{id}]"),
+ _ => format!("{} (not in the tree above)", complete(&focus, fetch(&focus), None).props.describe().0),
+ };
+ lines.push((Key::Info(0), format!("Focus: {what}")));
+ if let Some(sel) = str_attr(&focus, "AXSelectedText") {
+ lines.push((Key::Info(1), format!("Selected text: \"{}\"", tree::clip(&sel, tree::MAX_TEXT))));
}
- walk.visit(&c, 0, 0);
}
- let screenshot = if shot { Some(capture(&w, f)?) } else { None };
- let base = prev.filter(|_| diff);
- let tree: String = walk
- .lines
- .iter()
- .filter(|(id, l)| base.as_ref().is_none_or(|b| b.lines.get(id) != Some(l)))
- .map(|(_, l)| format!("{l}\n"))
- .collect();
- let removed = base
- .as_ref()
- .map(|b| {
- let mut gone: Vec<u32> = b.lines.keys().filter(|id| !walk.taken.contains(id)).copied().collect();
- gone.sort_unstable();
- gone
- })
- .unwrap_or_default();
+ let screenshot = if shot {
+ let related = input::related_windows(app.pid, &w, f, &others);
+ Some(capture(&w, f, &related)?)
+ } else {
+ None
+ };
+ let moved: HashSet<u32> = moved.into_values().collect();
+ let rendered = tree::render(&lines, prev.as_ref().filter(|_| diff).map(|p| &p.lines), &moved);
let obs = Observation {
app: app.name.clone(),
bundle_id: app.bundle.clone(),
pid: app.pid,
window: str_attr(&w, "AXTitle").unwrap_or_default(),
+ window_id: win,
frame: f,
- elements: walk.elements.len() as u32,
- truncated: walk.truncated,
- diffed: base.is_some(),
- tree,
- removed,
+ elements: lines.iter().filter(|(k, _)| matches!(k, Key::Element(_))).count() as u32,
+ truncated: walk.truncated || cut,
+ diffed: rendered.diffed,
+ tree: rendered.tree,
+ removed: rendered.removed,
screenshot,
};
+ let lines: HashMap<_, _> = lines.into_iter().collect();
self.cache.insert(
app.pid,
Snapshot {
window: win,
window_element: w.clone(),
- origin: walk.origin,
+ frame: f,
elements: walk.elements,
- lines: walk.lines.into_iter().collect(),
- next: walk.next,
+ popups: walk.popups,
+ text: walk.text.into_iter().filter(|(id, _)| lines.contains_key(&Key::Element(*id))).collect(),
+ markers: walk.markers.into_iter().filter(|(id, _)| lines.contains_key(&Key::Element(*id))).collect(),
+ ancestry: walk.ancestry.into_iter().filter(|(id, _)| lines.contains_key(&Key::Element(*id))).collect(),
+ lines,
+ ids: walk.ids.carry(),
},
);
Ok(obs)
}
- fn settle(&self, w: &El) {
- cancellation::wait(Duration::from_millis(1000));
- for _ in 0..16 {
- if cancellation::cancelled() || !busy(w, 0, &mut 0) {
- return;
- }
- std::thread::sleep(Duration::from_millis(300));
- }
- }
-
fn act(
&mut self,
query: &str,
@@ -634,27 +1727,93 @@ impl Desk {
mut pace: Option<&mut dyn FnMut(u32, Point) -> Result<(), Failure>>,
) -> Result<Reply, Failure> {
let app = resolve(query)?;
+ self.sync_app(&app);
if !self.cache.contains_key(&app.pid) {
return Err(fail(ErrorCode::BadIndex, "Observe this app before acting; its previous observation is no longer available"));
}
let (ax, w) = window(&app)?;
let win = window_id(&w);
+ if !on_current_space(win) {
+ return Err(fail(ErrorCode::NoWindow, "Window is not visible on the current Space; switch Spaces and observe again"));
+ }
+ self.validate_window(&app)?;
+ let mut focus = if actions.iter().any(|a| !matches!(a, Action::Wait { .. } | Action::Raise)) {
+ Some(crate::focus::Focus::begin(app.pid, win, std::rc::Rc::clone(&self.enforcers))?)
+ } else {
+ None
+ };
let mut completed = 0;
let mut failure = None;
+ let mut previous = Instant::now();
for (i, a) in actions.iter().enumerate() {
- let paced = match (pace.as_mut(), self.aim(app.pid, &w, a)) {
- (Some(p), Ok(Some(at))) => p(i as u32, Point { x: at.x, y: at.y }),
- (Some(_), Err(e)) => Err(e),
- _ => Ok(()),
- };
- if let Err(mut e) = paced.and_then(|_| self.validate_window(&app)).and_then(|_| self.step(&app, &ax, &w, win, a)) {
+ if i > 0 && !matches!(a, Action::Wait { .. }) && !matches!(actions[i - 1], Action::Wait { .. }) {
+ if let Some(o) = self.observers.get(&app.pid) {
+ quiet(previous, &o.notices, Duration::from_millis(60), Duration::from_millis(120), Duration::from_millis(800));
+ } else {
+ cancellation::wait(Duration::from_millis(120).saturating_sub(previous.elapsed()));
+ }
+ }
+ if matches!(a, Action::Raise) {
+ if let Some(mut guard) = focus.take() {
+ if let Err(mut error) = guard.finish() {
+ error.action = Some(i as u32);
+ failure = Some(error);
+ break;
+ }
+ }
+ if let Err(mut error) = self.enforcers.remove(app.pid, true) {
+ error.action = Some(i as u32);
+ failure = Some(error);
+ break;
+ }
+ }
+ let paced = focus
+ .as_ref()
+ .map_or(Ok(()), |f| f.check())
+ .and_then(|_| self.validate_window(&app))
+ .and_then(|_| self.check_menu(app.pid, &ax, a))
+ .and_then(|_| match pace.as_mut() {
+ Some(p) => match self.aim(app.pid, &w, a) {
+ Ok(Some(at)) => p(i as u32, Point { x: at.x, y: at.y }),
+ Ok(None) => Ok(()),
+ Err(e) => Err(e),
+ },
+ None => Ok(()),
+ });
+ if let Err(mut e) = paced
+ .and_then(|_| self.validate_window(&app))
+ .and_then(|_| self.step(&app, &ax, &w, a))
+ .and_then(|_| focus.as_ref().map_or(Ok(()), |f| f.check()))
+ {
e.action = Some(i as u32);
failure = Some(e);
break;
}
completed += 1;
+ previous = Instant::now();
+ }
+ if let Some(mut guard) = focus.take() {
+ if let Err(error) = guard.finish() {
+ if let Some(failure) = &mut failure {
+ failure.message += &format!("; background focus cleanup also failed: {}", error.message);
+ } else {
+ failure = Some(error);
+ }
+ }
+ }
+ let attempted = (completed as usize + 1).min(actions.len());
+ if actions[..attempted].iter().any(|a| !matches!(a, Action::Wait { .. })) {
+ self.touched.insert(app.pid, Instant::now());
+ }
+ if failure.is_some() || cancellation::cancelled() {
+ if let Err(error) = self.enforcers.clear() {
+ if let Some(failure) = failure.as_mut() {
+ failure.message.push_str(&format!("; cleanup: {}", error.message));
+ } else {
+ failure = Some(error);
+ }
+ }
}
- self.settle(&w);
let (observation, observation_failure) = match observe {
Some((shot, diff)) => match self.observe(&app, shot, diff) {
Ok(observation) => (Some(observation), None),
@@ -669,8 +1828,8 @@ impl Desk {
active()?;
let (_, current) = window(app)?;
let snapshot = self.cache.get(&app.pid).ok_or_else(|| fail(ErrorCode::BadIndex, "Observe before acting"))?;
- let origin = frame(&current).map(|f| (f.x, f.y));
- if current != snapshot.window_element || origin != Some(snapshot.origin) {
+ let current_id = window_id(&current);
+ if !snapshot.matches_window(&current, current_id, frame(&current)) || !on_current_space(current_id) {
return Err(fail(ErrorCode::BadIndex, "The window changed or moved; observe again before acting"));
}
Ok(())
@@ -678,28 +1837,121 @@ impl Desk {
fn element(&self, pid: i32, index: u32) -> Result<El, Failure> {
let s = self.cache.get(&pid).ok_or_else(|| fail(ErrorCode::BadIndex, "observe first"))?;
- s.elements
+ let e = s
+ .elements
.iter()
.find(|(id, _)| *id == index)
.map(|(_, e)| e.clone())
- .ok_or_else(|| fail(ErrorCode::BadIndex, format!("no element {index}; observe again")))
+ .ok_or_else(|| fail(ErrorCode::BadIndex, format!("no element {index}; observe again")))?;
+ let gone = || fail(ErrorCode::BadIndex, format!("element {index} changed or is gone; observe again before acting"));
+ let target = s.ids.locator(index).ok_or_else(gone)?;
+ let mut roots: Vec<(El, Locator)> =
+ attr(&s.window_element, "AXChildren").map(list).unwrap_or_default().into_iter().map(|c| (c, Locator::ROOT)).collect();
+ let extra_roots = extras(&app_element(pid), &s.window_element);
+ roots.extend(extra_roots.iter().cloned().map(|o| (o, Locator::ROOT)));
+ if !dead(&e) {
+ let mut ancestry = s.ancestry.get(&index).ok_or_else(gone)?.as_slice();
+ if let Some(at) = ancestry.iter().take(2).position(|element| extra_roots.contains(element)) {
+ let parent = attr(&ancestry[at], "AXParent");
+ if parent.as_ref() != at.checked_sub(1).map(|parent| &ancestry[parent]) {
+ return Err(gone());
+ }
+ ancestry = &ancestry[at..];
+ }
+ let current = tree::path_locator(&e, &roots, MAX_DEPTH, ancestry, |element| {
+ if cancellation::cancelled() {
+ return None;
+ }
+ let node = fetch(element);
+ Some((node.props, node.children))
+ });
+ return if current == Some(target) { Ok(e) } else { Err(gone()) };
+ }
+ s.ids.relocate(index, |locator| relocate(&roots, locator)).ok_or_else(gone)
}
fn point(&self, pid: i32, t: Target) -> Result<CGPoint, Failure> {
match t {
Target::Point(p) => {
- let o = self.cache.get(&pid).map(|s| s.origin).unwrap_or((0.0, 0.0));
+ let o = self.cache.get(&pid).map(|s| (s.frame.x, s.frame.y)).unwrap_or((0.0, 0.0));
Ok(pt(o.0 + p.x, o.1 + p.y))
}
Target::Index(i) => {
- let f = frame(&self.element(pid, i)?).ok_or_else(|| fail(ErrorCode::BadIndex, format!("element {i} has no frame")))?;
- Ok(pt(f.x + f.w / 2.0, f.y + f.h / 2.0))
+ let e = self.element(pid, i)?;
+ let window = self.cache.get(&pid).and_then(|s| frame(&s.window_element));
+ let visible = || {
+ let f = frame(&e).ok_or_else(|| fail(ErrorCode::BadIndex, format!("element {i} has no frame")))?;
+ Ok::<_, Failure>(window.map_or(Some(f), |w| visible_rect(&e, f, w)))
+ };
+ let r = match visible()? {
+ Some(r) => r,
+ None => {
+ let scrolled = perform(&app_element(pid), &e, "AXScrollToVisible");
+ cancellation::wait(Duration::from_millis(250));
+ visible()?.ok_or_else(|| {
+ let why = scrolled.err().map(|e| format!(" ({})", e.message)).unwrap_or_default();
+ fail(
+ ErrorCode::ActionFailed,
+ format!(
+ "element {i} is offscreen and could not be scrolled into view{why}; scroll toward it and observe again"
+ ),
+ )
+ })?
+ }
+ };
+ input::clickable(&app_element(pid), &e, i, r)
}
}
}
+ fn check_menu(&self, pid: i32, ax: &El, action: &Action) -> Result<(), Failure> {
+ input::check_menu_action(
+ action,
+ |index| {
+ let element = self.element(pid, index)?;
+ Ok(input::is_menu_item(&element) && actions(&element).iter().any(|action| action == "AXPress"))
+ },
+ || {
+ let snapshot = self.cache.get(&pid).ok_or_else(|| fail(ErrorCode::BadIndex, "Observe before acting"))?;
+ input::menu_open(ax, &snapshot.popups)
+ },
+ )
+ }
+
+ fn focus_field(&self, pid: i32, ax: &El, element: &El, index: u32) -> Result<(), Failure> {
+ input::focus_field(ax, element, || {
+ let snapshot = self.cache.get(&pid).ok_or_else(|| fail(ErrorCode::BadIndex, "Observe before acting"))?;
+ if input::menu_open(ax, &snapshot.popups)? {
+ return Err(fail(ErrorCode::ActionFailed, input::MENU_OPEN));
+ }
+ let point = self.point(pid, Target::Index(index))?;
+ let (owner, press) = self.pointer(ax, point)?;
+ click_at(owner, press, point, Button::Left, 1, &[], 0)
+ })
+ }
+
+ fn pointer(&self, ax: &El, p: CGPoint) -> Result<(i32, Press), Failure> {
+ let (owner, anchor, win, _) = input::service_at(ax, p)?;
+ Ok((owner, prepare_window(owner, ax, anchor, win, p)?))
+ }
+
+ fn pressed(&self, pid: i32, index: u32) -> Result<Option<El>, Failure> {
+ let e = self.element(pid, index)?;
+ let can_press = actions(&e).iter().any(|a| a == "AXPress");
+ let menu_item = input::is_menu_item(&e);
+ if menu_item && !can_press {
+ return Err(fail(ErrorCode::ActionFailed, "Menu item no longer supports AXPress; observe again"));
+ }
+ Ok((can_press && (menu_item || !input::simulate(&e) || frame(&e).is_none())).then_some(e))
+ }
+
fn aim(&self, pid: i32, w: &El, a: &Action) -> Result<Option<CGPoint>, Failure> {
match a {
+ Action::Click { target: Target::Index(i), button: Button::Left, count: 1, modifiers, hold_ms: 0 }
+ if modifiers.is_empty() && self.pressed(pid, *i)?.is_some() =>
+ {
+ Ok(None)
+ }
Action::Click { target, .. } | Action::Scroll { target: Some(target), .. } => self.point(pid, *target).map(Some),
Action::Scroll { target: None, .. } => Ok(Some(center(w))),
Action::Drag { from, .. } => self.point(pid, Target::Point(*from)).map(Some),
@@ -707,34 +1959,78 @@ impl Desk {
}
}
- fn step(&self, app: &App, ax: &El, w: &El, win: u32, a: &Action) -> Result<(), Failure> {
+ fn step(&self, app: &App, ax: &El, w: &El, a: &Action) -> Result<(), Failure> {
let pid = app.pid;
match a {
Action::Click { target: Target::Index(i), button: Button::Left, count: 1, modifiers, hold_ms: 0 } if modifiers.is_empty() => {
- let e = self.element(pid, *i)?;
- if actions(&e).iter().any(|a| a == "AXPress") {
- perform(&e, "AXPress")
- } else {
- click_at(pid, win, self.point(pid, Target::Index(*i))?, Button::Left, 1, &[], 0)
+ if let Some(e) = self.pressed(pid, *i)? {
+ return perform(ax, &e, "AXPress");
+ }
+ match self.point(pid, Target::Index(*i)) {
+ Ok(p) => {
+ let (to, press) = self.pointer(ax, p)?;
+ click_at(to, press, p, Button::Left, 1, &[], 0)
+ }
+ Err(e) => Err(e),
}
}
Action::Click { target, button, count, modifiers, hold_ms } => {
- click_at(pid, win, self.point(pid, *target)?, *button, *count, modifiers, *hold_ms)
+ let p = self.point(pid, *target)?;
+ let (to, press) = self.pointer(ax, p)?;
+ click_at(to, press, p, *button, *count, modifiers, *hold_ms)
}
- Action::TypeText { text } => {
- for c in text.chars() {
- key(pid, 0, CGEventFlags::empty(), Some(&c.to_string()), 0)?;
- }
- Ok(())
+ Action::TypeText { text } => type_text(input::keyboard_target(ax)?.0, text),
+ Action::Paste { text, format } => {
+ let (to, target) = input::keyboard_target(ax)?;
+ paste(to, &target, text, *format)
}
Action::SelectText { index, text, prefix, suffix, selection } => {
- select(&self.element(pid, *index)?, text, prefix, suffix, *selection)
+ if self.cache.get(&pid).is_some_and(|snapshot| snapshot.markers.contains_key(index)) {
+ return Err(fail(
+ ErrorCode::ActionFailed,
+ "This element exposes text markers without a numeric text value; selection by text is unavailable",
+ ));
+ }
+ let element = self.element(pid, *index)?;
+ select(
+ || self.focus_field(pid, ax, &element, *index),
+ &element,
+ self.cache.get(&pid).and_then(|snapshot| snapshot.text.get(index)),
+ text,
+ prefix,
+ suffix,
+ *selection,
+ )
}
Action::PressKey { key: k, hold_ms } => {
- let (code, mods) = chord(k)?;
- key(pid, code, flags(&mods), None, *hold_ms)
+ let mut chord = crate::keys::parse(k).ok_or_else(|| fail(ErrorCode::UnknownKey, format!("unknown key {k}")))?;
+ if let Some(c) = chord.character {
+ if let Some(stroke) = keyboard_for_flags(CGEventFlags::from_bits_truncate(chord.flags)).get(&c) {
+ chord.remap(*stroke);
+ }
+ }
+ send_key_events(
+ input::keyboard_target(ax)?.0,
+ crate::keys::events_with_modifiers(chord.code, chord.flags, &chord.modifiers),
+ None,
+ *hold_ms,
+ )
}
Action::Scroll { target, direction, amount } => {
+ let mut viewport = None;
+ if let Amount::Pages(n) = amount {
+ let area = match target {
+ Some(Target::Index(i)) => input::scroll_area(&self.element(pid, *i)?),
+ Some(t) => input::area_at(ax, self.point(pid, *t)?),
+ None => input::area_at(ax, center(w)),
+ };
+ if let Some(area) = area {
+ viewport = frame(&area);
+ if input::scroll_pages(ax, &area, *direction, *n)? {
+ return Ok(());
+ }
+ }
+ }
let p = match target {
Some(t) => self.point(pid, *t)?,
None => center(w),
@@ -744,11 +2040,8 @@ impl Desk {
_ => frame(w),
}
.unwrap_or(Rect { x: 0.0, y: 0.0, w: 600.0, h: 600.0 });
- let vertical = matches!(direction, Direction::Up | Direction::Down);
- let mut left = match amount {
- Amount::Pixels(n) => *n as i32,
- Amount::Pages(n) => (n.clamp(0.0, 20.0) * 0.85 * if vertical { view.h } else { view.w }) as i32,
- };
+ let (to, press) = self.pointer(ax, p)?;
+ let mut left = input::scroll_pixels(*amount, *direction, viewport, view);
while left > 0 {
active()?;
let n = left.min(120);
@@ -762,43 +2055,57 @@ impl Desk {
let e = CGEvent::new_scroll_event(source()?, ScrollEventUnit::PIXEL, 2, dy, dx, 0)
.map_err(|_| fail(ErrorCode::ActionFailed, "scroll event"))?;
unsafe { CGEventSetLocation(e.as_ptr() as *mut c_void, p) };
- post(e, pid);
+ validate_input_window(&press)?;
+ in_window(&e, press.win, &press.frame, p)?;
+ active()?;
+ post(e, to);
}
Ok(())
}
Action::Drag { from, to } => {
let (a, b) = (self.point(pid, Target::Point(*from))?, self.point(pid, Target::Point(*to))?);
let none = CGEventFlags::empty();
- mouse(pid, win, CGEventType::LeftMouseDown, a, CGMouseButton::Left, 1, none)?;
+ let (pid, press) = self.pointer(ax, a)?;
+ mouse(pid, &press, CGEventType::LeftMouseDown, a, CGMouseButton::Left, 1, none)?;
let mut last = a;
let dragged = (|| {
+ cancellation::wait(Duration::from_millis(PRESS_MS));
for s in 1..=12 {
active()?;
let t = s as f64 / 12.0;
let point = pt(a.x + (b.x - a.x) * t, a.y + (b.y - a.y) * t);
- mouse(pid, win, CGEventType::LeftMouseDragged, point, CGMouseButton::Left, 1, none)?;
+ mouse(pid, &press, CGEventType::LeftMouseDragged, point, CGMouseButton::Left, 1, none)?;
last = point;
cancellation::wait(Duration::from_millis(12));
}
Ok(())
})();
- let released = mouse(pid, win, CGEventType::LeftMouseUp, last, CGMouseButton::Left, 1, none);
+ let released = mouse(pid, &press, CGEventType::LeftMouseUp, last, CGMouseButton::Left, 1, none);
dragged.and(released)
}
- Action::SetValue { index, value } => match set_attr(&self.element(pid, *index)?, "AXValue", &cfs(value).as_CFType()) {
- 0 => Ok(()),
- n => Err(fail(ErrorCode::ActionFailed, format!("setting AXValue failed (AXError {n})"))),
- },
+ Action::SetValue { index, value, submit } => {
+ let element = self.element(pid, *index)?;
+ input::set_value(pid, &element, *index, value, *submit, || self.focus_field(pid, ax, &element, *index))
+ }
Action::Perform { index, action } => {
let name = if action.starts_with("AX") { action.clone() } else { format!("AX{action}") };
- perform(&self.element(pid, *index)?, &name)
+ let element = self.element(pid, *index)?;
+ let listed = actions(&element);
+ if !listed.contains(&name) {
+ return Err(fail(
+ ErrorCode::ActionFailed,
+ format!("action {name} is not listed for element {index}; listed actions: {}", listed.join(", ")),
+ ));
+ }
+ perform(ax, &element, &name)
}
Action::Raise => {
+ input::register_action_target(ax, w)?;
let status = set_attr(ax, "AXFrontmost", &CFBoolean::true_value().as_CFType());
if status != 0 {
return Err(fail(ErrorCode::ActionFailed, format!("setting AXFrontmost failed (AXError {status})")));
}
- perform(w, "AXRaise")
+ perform(ax, w, "AXRaise")
}
Action::Wait { ms } => {
cancellation::wait(Duration::from_millis((*ms).min(10_000) as u64));
@@ -809,7 +2116,7 @@ impl Desk {
}
fn apps() -> Result<Reply, Failure> {
- let apps = running().into_iter().map(|a| AppInfo { name: a.name, bundle_id: a.bundle, pid: a.pid, active: a.active }).collect();
+ let apps = running(false).into_iter().map(|a| AppInfo { name: a.name, bundle_id: a.bundle, pid: a.pid, active: a.active }).collect();
Ok(Reply::Apps { apps })
}
@@ -817,48 +2124,364 @@ fn resolve(query: &str) -> Result<App, Failure> {
if !unsafe { AXIsProcessTrusted() } {
return Err(fail(ErrorCode::PermissionMissing, "Accessibility not granted; run `desk permissions --prompt`"));
}
- let find = || {
- running()
- .into_iter()
- .find(|a| a.name.eq_ignore_ascii_case(query) || a.bundle.as_deref().is_some_and(|b| b.eq_ignore_ascii_case(query)))
+ let path = if std::path::Path::new(query).is_absolute() {
+ Some(
+ std::fs::canonicalize(query)
+ .map_err(|error| fail(ErrorCode::AppNotFound, format!("Cannot resolve app path {query}: {error}")))?,
+ )
+ } else {
+ None
};
- let app = match find() {
- Some(app) => app,
- None => {
- let flag = if query.contains('.') && !query.contains(' ') { "-b" } else { "-a" };
- let opened = std::process::Command::new("/usr/bin/open").args(["-g", flag, query]).status().is_ok_and(|s| s.success());
- (0..25)
- .take_while(|_| opened)
- .find_map(|_| {
- std::thread::sleep(Duration::from_millis(200));
- find()
- })
- .ok_or_else(|| fail(ErrorCode::AppNotFound, format!("could not open {query}")))?
+ let find = || select_app(running(true), query, path.as_deref());
+ if find()?.is_none() {
+ let flag = if path.is_none() && query.contains('.') && !query.contains(' ') { "-b" } else { "-a" };
+ let opened = std::process::Command::new("/usr/bin/open").args(["-g", flag, query]).status().is_ok_and(|s| s.success());
+ if !opened {
+ return Err(fail(ErrorCode::AppNotFound, format!("could not open {query}")));
}
- };
- Ok(app)
+ }
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ active()?;
+ let pending = match find()? {
+ Some(app) => {
+ let launched =
+ NSRunningApplication::runningApplicationWithProcessIdentifier(app.pid).is_some_and(|app| app.isFinishedLaunching());
+ if launched {
+ match window(&app) {
+ Ok(_) => return Ok(app),
+ Err(error) => error,
+ }
+ } else {
+ fail(ErrorCode::NoWindow, format!("{} did not finish launching", app.name))
+ }
+ }
+ None => fail(ErrorCode::AppNotFound, format!("{query} did not start or exited before becoming ready")),
+ };
+ if Instant::now() >= deadline {
+ return Err(pending);
+ }
+ CFRunLoop::run_in_mode(unsafe { kCFRunLoopDefaultMode }, Duration::from_millis(100), false);
+ }
+}
+
+fn select_app(apps: Vec<App>, query: &str, path: Option<&std::path::Path>) -> Result<Option<App>, Failure> {
+ let mut matches: Vec<_> = apps
+ .into_iter()
+ .filter(|app| match path {
+ Some(path) => app.path.as_deref() == Some(path),
+ None => app.name.eq_ignore_ascii_case(query) || app.bundle.as_deref().is_some_and(|bundle| bundle.eq_ignore_ascii_case(query)),
+ })
+ .collect();
+ if matches.len() > 1 {
+ matches.sort_by_key(|app| app.pid);
+ let candidates = matches
+ .iter()
+ .map(|app| {
+ format!(
+ "{} (PID {}, {}, {})",
+ app.name,
+ app.pid,
+ app.bundle.as_deref().unwrap_or("no bundle ID"),
+ app.path.as_ref().map(|path| path.display().to_string()).unwrap_or_else(|| "unknown path".into())
+ )
+ })
+ .collect::<Vec<_>>()
+ .join("; ");
+ return Err(fail(
+ ErrorCode::BadRequest,
+ format!(
+ "Ambiguous app {query:?}: {candidates}. Use a unique bundle ID or full application path, or close duplicate instances."
+ ),
+ ));
+ }
+ Ok(matches.pop())
}
fn window(app: &App) -> Result<(El, El), Failure> {
let ax = unsafe { CFType::wrap_under_create_rule(AXUIElementCreateApplication(app.pid)) };
unsafe { AXUIElementSetMessagingTimeout(ax.as_CFTypeRef(), 1.5) };
- set_attr(&ax, "AXManualAccessibility", &CFBoolean::true_value().as_CFType());
let w = window_of(&ax).ok_or_else(|| fail(ErrorCode::NoWindow, format!("{} has no open window", app.name)))?;
+ if bool_attr(&ax, "AXHidden") == Some(true) || bool_attr(&w, "AXMinimized") == Some(true) {
+ return Err(fail(ErrorCode::NoWindow, "App is hidden or window is minimized; show it and observe again"));
+ }
Ok((ax, w))
}
-fn capture(w: &El, f: Rect) -> Result<Screenshot, Failure> {
- if !unsafe { CGPreflightScreenCaptureAccess() } {
+fn capture(w: &El, f: Rect, panels: &[u32]) -> Result<Screenshot, Failure> {
+ if !CGPreflightScreenCaptureAccess() {
return Err(fail(ErrorCode::PermissionMissing, "Screen Recording not granted; run `desk permissions --prompt`"));
}
- let rect = CGRect::new(&pt(f.x, f.y), &CGSize::new(f.w, f.h));
- let img = create_image(
- rect,
- kCGWindowListOptionIncludingWindow,
- window_id(w),
- kCGWindowImageBoundsIgnoreFraming | kCGWindowImageNominalResolution,
- )
- .ok_or_else(|| fail(ErrorCode::ActionFailed, "window capture returned nothing"))?;
- let (png_base64, width, height) = png(&img).ok_or_else(|| fail(ErrorCode::ActionFailed, "unexpected pixel format"))?;
- Ok(Screenshot { png_base64, width, height, pixels_per_point: if f.w > 0.0 { width as f64 / f.w } else { 1.0 } })
+ crate::capture::window(window_id(w), panels, f).map_err(|m| fail(ErrorCode::ActionFailed, m))
+}
+
+pub(crate) fn input_window_geometry(id: u32) -> Option<(i32, Rect)> {
+ input::window_geometry(id)
+}
+
+pub(crate) fn menu_observer(pid: i32, state: std::sync::Arc<crate::intervention::RetainedState>) -> Option<Observer> {
+ let app = app_element(pid);
+ if unsafe { AXUIElementSetMessagingTimeout(app.as_CFTypeRef(), 0.08) } != 0 {
+ return None;
+ }
+ let mut observer = Observer::with_notices(pid, &app, &["AXMenuOpened", "AXMenuClosed", "AXUIElementDestroyed"])?;
+ observer.notices.pid = pid;
+ observer.notices.menu = Some(state);
+ Some(observer)
+}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn web_readiness_waits_for_missing_electron_content_and_stops_at_limits() {
+ use super::web_ready;
+ use std::collections::HashMap;
+ let mut nodes = HashMap::from([(0, (false, vec![1])), (1, (false, vec![]))]);
+ assert_eq!(web_ready(0, false, |id| nodes.get(id).cloned()), Some(true));
+ assert_eq!(web_ready(0, true, |id| nodes.get(id).cloned()), Some(false));
+ nodes.insert(1, (true, vec![]));
+ assert_eq!(web_ready(0, true, |id| nodes.get(id).cloned()), Some(false));
+ nodes.insert(1, (true, vec![2]));
+ assert_eq!(web_ready(0, true, |id| nodes.get(id).cloned()), Some(true));
+ assert_eq!(web_ready(0, true, |_| None::<(bool, Vec<i32>)>), None);
+ assert_eq!(web_ready(0, true, |id| Some((false, vec![id + 1]))), None);
+ }
+
+ #[test]
+ fn primary_window_preserves_authoritative_choice_and_uses_last_fallback() {
+ use super::primary_window;
+ assert_eq!(primary_window(Some((1, true)), || panic!("focused window is authoritative"), || vec![(2, false)]), Some((1, true)));
+ assert_eq!(primary_window(None, || Some(2), || panic!("main window is authoritative")), Some(2));
+ assert_eq!(primary_window(None, || None, || vec![1, 2, 3]), Some(3));
+ assert_eq!(primary_window::<u32>(None, || None, Vec::new), None);
+ }
+
+ #[test]
+ fn copied_ax_results_release_owned_values_on_error_and_success() {
+ let value = CFArray::from_CFTypes(&[cfs("owned AX result")]);
+ let baseline = value.retain_count();
+ for status in [0, -25200] {
+ let owned = value.clone();
+ let pointer = owned.as_CFTypeRef();
+ std::mem::forget(owned);
+ let result = ax_result(status, pointer);
+ assert_eq!(result.is_some(), status == 0);
+ drop(result);
+ assert_eq!(value.retain_count(), baseline);
+ }
+ assert!(ax_result(0, std::ptr::null()).is_none());
+ }
+
+ #[test]
+ fn observed_window_rejects_rebound_identity_with_same_element_and_frame() {
+ let element = app_element(10001);
+ let frame = Rect { x: 0.0, y: 0.0, w: 100.0, h: 100.0 };
+ let snapshot = Snapshot {
+ window: 42,
+ window_element: element.clone(),
+ frame,
+ elements: vec![],
+ popups: vec![],
+ text: HashMap::new(),
+ markers: HashMap::new(),
+ ancestry: HashMap::new(),
+ lines: HashMap::new(),
+ ids: tree::Ids::default(),
+ };
+ assert!(snapshot.matches_window(&element, 42, Some(frame)));
+ assert!(!snapshot.matches_window(&element, 43, Some(frame)));
+ assert!(!snapshot.matches_window(&element, 0, Some(frame)));
+ }
+
+ #[test]
+ fn prefetched_children_keep_content_clipping_and_visit_budget() {
+ let window = Rect { x: 0.0, y: 0.0, w: 100.0, h: 100.0 };
+ let mut walk = Walk {
+ ids: tree::Ids::default(),
+ elements: vec![],
+ popups: vec![],
+ text: HashMap::new(),
+ markers: HashMap::new(),
+ ancestry: HashMap::new(),
+ text_budget: 1024 * 1024,
+ notes: HashMap::new(),
+ truncated: false,
+ visited: MAX_VISITS,
+ ancestors: vec![],
+ window,
+ cache: None,
+ };
+ let node = |title: &str, x| Node {
+ has_value: false,
+ props: Props { role: "AXButton".into(), title: Some(title.into()), ..Default::default() },
+ raw: None,
+ rendered: None,
+ frame: Some(Rect { x, y: 10.0, w: 20.0, h: 20.0 }),
+ children: vec![],
+ };
+ let elements = [app_element(10001), app_element(10002)];
+ let mut items = vec![];
+ walk.children(
+ &elements,
+ Some(vec![node("Apply", 10.0), node("Offscreen", 200.0)]),
+ WalkAt { depth: 0, anchor: ROOT, clip: window, at: Locator::ROOT },
+ false,
+ &mut items,
+ );
+ let (lines, cut) = tree::layout(&items, MAX_TREE_CHARS);
+ assert_eq!(walk.visited, MAX_VISITS);
+ assert!(!walk.truncated && !cut);
+ assert_eq!(walk.elements.len(), 1);
+ assert!(lines.iter().any(|(_, line)| line.contains("Apply")));
+ assert!(lines.iter().any(|(_, line)| line.contains("1 more offscreen")));
+ assert!(!lines.iter().any(|(_, line)| line.contains("Offscreen")));
+ let mut unprefetched = vec![];
+ walk.children(&elements[..1], None, WalkAt { depth: 0, anchor: ROOT, clip: window, at: Locator::ROOT }, false, &mut unprefetched);
+ assert!(walk.truncated);
+ assert_eq!(walk.visited, MAX_VISITS);
+ }
+
+ #[test]
+ fn marker_candidates_require_absent_native_value() {
+ let clip = Rect { x: 0.0, y: 0.0, w: 100.0, h: 100.0 };
+ for value in [None, Some(""), Some(" "), Some("text")] {
+ let node = Node {
+ has_value: value.is_some(),
+ props: Props {
+ role: "AXTextField".into(),
+ value: value.map(str::trim).filter(|s| !s.is_empty()).map(str::to_string),
+ ..Default::default()
+ },
+ raw: value.map(str::to_string),
+ rendered: None,
+ frame: Some(clip),
+ children: vec![],
+ };
+ assert_eq!(node.marker_candidate(clip), value.is_none());
+ }
+ }
+
+ #[test]
+ fn observed_selection_follows_rebound_element_ids() {
+ let locator = Locator::ROOT.child(&Props { role: "AXTextArea".into(), identifier: Some("editor".into()), ..Default::default() });
+ let mut ids = tree::Ids::default();
+ let old = ids.assign(&"old", 1, locator);
+ let mut ids = ids.carry();
+ let new = ids.assign(&"new", 2, locator);
+ let rendered = text::attributed_text(&[("hello".into(), text::TextStyle { bold: true, ..Default::default() })], true).unwrap();
+ let mapping = rebind_text(HashMap::from([(new, text::ObservedText::new("hello", rendered))]), &ids.rebind(|e| *e == "old"));
+ assert!(!mapping.contains_key(&new));
+ assert_eq!(text::observed_selection_range("hello", mapping.get(&old), "**hello**", "", "", Selection::Text).unwrap(), [0, 5]);
+ }
+
+ #[test]
+ fn observed_selection_budget_falls_back_to_plain_text() {
+ let rendered = text::attributed_text(&[("hello".into(), text::TextStyle { bold: true, ..Default::default() })], true).unwrap();
+ let mut budget = rendered.retained_bytes();
+ let mut node = Node {
+ has_value: true,
+ props: Props {
+ role: "AXTextArea".into(),
+ value: Some("hello".into()),
+ rich_text: Some(rendered.markdown.clone()),
+ ..Default::default()
+ },
+ raw: Some("hello".into()),
+ rendered: Some(rendered),
+ frame: None,
+ children: vec![],
+ };
+ node.limit_text(&mut budget);
+ assert_eq!(budget, 0);
+ assert!(node.props.describe().0.contains("markdown:"));
+ node.limit_text(&mut budget);
+ assert!(node.rendered.is_none());
+ assert_eq!(node.props.describe().0, "TextArea = \"hello\"");
+ }
+
+ #[test]
+ fn attributed_observation_resolves_native_font_traits() {
+ use core_foundation::attributed_string::CFMutableAttributedString;
+ for (name, expected) in [(".AppleSystemUIFontBold", "**word**"), (".SFNS-RegularItalic", "*word*")] {
+ let mut value = CFMutableAttributedString::new();
+ value.replace_str(&cfs("word"), CFRange { location: 0, length: 0 });
+ let font = CFDictionary::from_CFType_pairs(&[
+ (cfs("AXFontName"), cfs(name).as_CFType()),
+ (cfs("AXFontSize"), CFNumber::from(14.0).as_CFType()),
+ ]);
+ value.set_attribute(CFRange { location: 0, length: 4 }, cfs("AXFont").as_concrete_TypeRef(), &font);
+ let immutable = value.as_CFType().downcast::<CFAttributedString>().unwrap();
+ let runs = attributed_runs(&immutable, "word").unwrap();
+ assert_eq!(text::attributed_markdown(&runs, true).as_deref(), Some(expected), "{name}");
+ value.set_attribute(CFRange { location: 0, length: 4 }, cfs("AXFontBold").as_concrete_TypeRef(), &CFBoolean::false_value());
+ value.set_attribute(CFRange { location: 0, length: 4 }, cfs("AXFontItalic").as_concrete_TypeRef(), &CFBoolean::false_value());
+ let runs = attributed_runs(&immutable, "word").unwrap();
+ assert!(text::attributed_markdown(&runs, true).is_none());
+ }
+ }
+
+ #[test]
+ fn attributed_observation_decodes_native_runs_and_rejects_mismatches() {
+ use core_foundation::attributed_string::CFMutableAttributedString;
+ let mut value = CFMutableAttributedString::new();
+ value.replace_str(&cfs("😀bold\nplain"), CFRange { location: 0, length: 0 });
+ value.set_attribute(CFRange { location: 2, length: 4 }, cfs("AXFontBold").as_concrete_TypeRef(), &CFBoolean::true_value());
+ let value = value.as_CFType().downcast::<CFAttributedString>().unwrap();
+ let runs = attributed_runs(&value, "😀bold\nplain").unwrap();
+ assert_eq!(text::attributed_markdown(&runs, true).unwrap(), "😀**bold**\nplain");
+ assert!(attributed_runs(&value, "stale text").is_none());
+ assert_eq!(text::selection_range("😀bold\nplain", "bold", "", "", Selection::Text).unwrap(), [2, 4]);
+ }
+
+ use super::*;
+
+ fn app(name: &str, bundle: &str, path: &str, pid: i32) -> App {
+ App { name: name.into(), bundle: Some(bundle.into()), path: Some(path.into()), pid, active: false, launched: None }
+ }
+
+ #[test]
+ fn disjoint_scroll_clips_do_not_reveal_descendants() {
+ let window = Rect { x: 0.0, y: 0.0, w: 100.0, h: 100.0 };
+ let outer = narrow(window, "AXScrollArea", Some(Rect { x: 0.0, y: 0.0, w: 30.0, h: 30.0 }));
+ let inner = narrow(outer, "AXWebArea", Some(Rect { x: 50.0, y: 50.0, w: 40.0, h: 40.0 }));
+ let child = Rect { x: 5.0, y: 5.0, w: 10.0, h: 10.0 };
+ assert!(tree::visible(child, inner).is_none());
+ assert!(tree::visible(child, narrow(inner, "AXScrollArea", Some(window))).is_none());
+ }
+
+ #[test]
+ fn app_identity_allows_accessory_only_for_explicit_resolution() {
+ use NSApplicationActivationPolicy as Policy;
+ let candidates = |explicit| {
+ [(Policy::Regular, "Editor", 11), (Policy::Accessory, "Panel", 22), (Policy::Prohibited, "Helper", 33)]
+ .into_iter()
+ .filter(|(policy, _, _)| app_policy_allowed(*policy, explicit))
+ .map(|(_, name, pid)| app(name, &format!("test.{name}"), &format!("/{name}.app"), pid))
+ .collect::<Vec<_>>()
+ };
+ assert_eq!(candidates(false).iter().map(|app| app.pid).collect::<Vec<_>>(), [11]);
+ assert_eq!(select_app(candidates(true), "/Panel.app", Some(std::path::Path::new("/Panel.app"))).unwrap().unwrap().pid, 22);
+ assert_eq!(select_app(candidates(true), "test.Panel", None).unwrap().unwrap().pid, 22);
+ assert!(select_app(candidates(true), "Helper", None).unwrap().is_none());
+ assert!(select_app(candidates(true), "Pane", None).unwrap().is_none());
+ }
+
+ #[test]
+ fn app_identity_rejects_every_ambiguous_match() {
+ let result = select_app(vec![app("Editor", "test.one", "/One.app", 11), app("Editor", "test.two", "/Two.app", 22)], "editor", None);
+ let error = result.err().unwrap();
+ assert_eq!(error.code, ErrorCode::BadRequest);
+ assert!(error.message.contains("PID 11"));
+ assert!(error.message.contains("PID 22"));
+ let result = select_app(vec![app("One", "test.same", "/One.app", 11), app("Two", "test.same", "/Two.app", 22)], "test.same", None);
+ assert!(result.is_err());
+ }
+
+ #[test]
+ fn app_identity_selects_exact_path_or_unique_bundle() {
+ let apps = || vec![app("Editor", "test.one", "/One.app", 11), app("Editor", "test.two", "/Two.app", 22)];
+ assert_eq!(select_app(apps(), "test.TWO", None).unwrap().unwrap().pid, 22);
+ assert_eq!(select_app(apps(), "/One.app", Some(std::path::Path::new("/One.app"))).unwrap().unwrap().pid, 11);
+ assert!(select_app(apps(), "/Missing.app", Some(std::path::Path::new("/Missing.app"))).unwrap().is_none());
+ }
}
diff --git a/desktop/desk/src/mac/input.rs b/desktop/desk/src/mac/input.rs
new file mode 100644
index 000000000..3db724004
--- /dev/null
+++ b/desktop/desk/src/mac/input.rs
@@ -0,0 +1,800 @@
+use super::*;
+use crate::tree::clip;
+use core_graphics::window::{create_description_from_array, kCGWindowBounds, kCGWindowOwnerPID};
+
+const MENU_ITEMS: &[&str] = &["AXMenuItem", "AXMenuBarItem"];
+const SELECTS: &[&str] = &["AXRow", "AXCell", "AXOutlineRow", "AXListItem"];
+const TEXT_ROLES: &[&str] = &["AXTextField", "AXTextArea", "AXComboBox"];
+
+fn element_at(root: &El, p: CGPoint) -> Option<El> {
+ let mut v: CFTypeRef = std::ptr::null();
+ let status = unsafe { AXUIElementCopyElementAtPosition(root.as_CFTypeRef(), p.x as f32, p.y as f32, &mut v) };
+ ax_result(status, v)
+}
+
+pub(super) fn within(e: &El, ancestor: &El) -> bool {
+ let mut cur = Some(e.clone());
+ for _ in 0..MAX_DEPTH {
+ match cur {
+ Some(c) if c == *ancestor => return true,
+ Some(c) => cur = attr(&c, "AXParent"),
+ None => return false,
+ }
+ }
+ false
+}
+
+fn has_role(e: &El, roles: &[&str]) -> bool {
+ str_attr(e, "AXRole").is_some_and(|r| roles.contains(&r.as_str()))
+}
+
+fn describe(e: &El) -> String {
+ let role = str_attr(e, "AXRole").unwrap_or_default();
+ let name = str_attr(e, "AXTitle").or_else(|| str_attr(e, "AXDescription")).or_else(|| str_attr(e, "AXValue"));
+ match name {
+ Some(n) => format!("{} \"{}\"", role.trim_start_matches("AX"), clip(&n, 40)),
+ None => role.trim_start_matches("AX").to_string(),
+ }
+}
+
+fn clickable_point<T>(
+ visible: Rect,
+ mut hit: impl FnMut(CGPoint) -> Option<T>,
+ mut is_target: impl FnMut(&T) -> bool,
+) -> Result<CGPoint, Option<T>> {
+ let mut obstruction = None;
+ for (fx, fy) in [(0.5, 0.5), (0.25, 0.5), (0.75, 0.5), (0.5, 0.25), (0.5, 0.75)] {
+ let p = pt(visible.x + visible.w * fx, visible.y + visible.h * fy);
+ if let Some(element) = hit(p) {
+ if is_target(&element) {
+ return Ok(p);
+ }
+ obstruction.get_or_insert(element);
+ }
+ }
+ Err(obstruction)
+}
+
+pub(super) fn clickable(ax: &El, e: &El, index: u32, visible: Rect) -> Result<CGPoint, Failure> {
+ clickable_point(visible, |p| element_at(ax, p), |hit| within(hit, e)).map_err(|hit| {
+ let detail = hit.map(|element| format!("; hit testing found {} instead", describe(&element))).unwrap_or_default();
+ fail(ErrorCode::ActionFailed, format!("Cannot verify a clickable point for element {index}{detail}; reveal it and observe again"))
+ })
+}
+
+pub(super) fn window_geometry(id: u32) -> Option<(i32, Rect)> {
+ if id == 0 {
+ return None;
+ }
+ let windows = create_description_from_array(CFArray::from_copyable(&[id]))?;
+ let window = windows.get(0)?;
+ let owner = window.find(unsafe { CFString::wrap_under_get_rule(kCGWindowOwnerPID) })?.downcast::<CFNumber>()?.to_i64()?;
+ let owner = i32::try_from(owner).ok().filter(|pid| *pid > 0)?;
+ let bounds = window.find(unsafe { CFString::wrap_under_get_rule(kCGWindowBounds) })?.downcast::<CFDictionary>()?;
+ let bounds = core_graphics::geometry::CGRect::from_dict_representation(&bounds)?;
+ Some((owner, Rect { x: bounds.origin.x, y: bounds.origin.y, w: bounds.size.width, h: bounds.size.height }))
+}
+
+fn ancestor_window<T: Clone + PartialEq>(
+ host: &T,
+ element: &T,
+ mut parent: impl FnMut(&T) -> Option<T>,
+ mut native_window: impl FnMut(&T) -> u32,
+) -> Option<u32> {
+ let mut seen = Vec::new();
+ let mut current = element.clone();
+ let mut id = 0;
+ for _ in 0..MAX_DEPTH {
+ if seen.contains(&current) {
+ return None;
+ }
+ if id == 0 {
+ id = native_window(&current);
+ }
+ if current == *host {
+ return Some(id);
+ }
+ let next = parent(&current)?;
+ seen.push(current);
+ current = next;
+ }
+ None
+}
+
+pub(super) fn resolved_window(host: &El, element: &El) -> Result<(i32, u32, Rect), Failure> {
+ let id = ancestor_window(host, element, |element| attr(element, "AXParent"), window_id)
+ .ok_or_else(|| fail(ErrorCode::ActionFailed, "The input target is no longer linked to this app; observe again"))?;
+ let (owner, bounds) =
+ window_geometry(id).ok_or_else(|| fail(ErrorCode::NoWindow, "Input window ownership is unavailable; observe again"))?;
+ Ok((owner, id, bounds))
+}
+
+pub(super) fn register_target(host: &El, element: &El) -> Result<i32, Failure> {
+ let (owner, _, _) = resolved_window(host, element)?;
+ crate::intervention::include_target(owner);
+ active()?;
+ Ok(owner)
+}
+
+pub(super) fn register_action_target(host: &El, element: &El) -> Result<(), Failure> {
+ if window_id(element) == 0 && is_menu_item(element) && within(element, host) {
+ if let Some(owner) = pid_of(host).filter(|owner| pid_of(element) == Some(*owner)) {
+ crate::intervention::include_target(owner);
+ return active();
+ }
+ }
+ register_target(host, element).map(|_| ())
+}
+
+fn inside(inner: Rect, outer: Rect) -> bool {
+ const SLACK: f64 = 4.0;
+ inner.x >= outer.x - SLACK
+ && inner.y >= outer.y - SLACK
+ && inner.x + inner.w <= outer.x + outer.w + SLACK
+ && inner.y + inner.h <= outer.y + outer.h + SLACK
+}
+
+pub(super) fn service_at(ax: &El, p: CGPoint) -> Result<(i32, El, u32, Rect), Failure> {
+ let element = element_at(ax, p).ok_or_else(|| fail(ErrorCode::ActionFailed, "Cannot identify the input target; observe again"))?;
+ let (owner, id, bounds) = resolved_window(ax, &element)?;
+ Ok((owner, element, id, bounds))
+}
+
+pub(super) fn related_windows(pid: i32, window: &El, outer: Rect, extras: &[El]) -> Vec<u32> {
+ let own = window_id(window);
+ let mut found = vec![];
+ let mut visited = 0;
+ for (root, contained) in extras.iter().map(|root| (root, false)).chain(std::iter::once((window, true))) {
+ let mut checked = HashSet::from([own]);
+ let mut queue = std::collections::VecDeque::from([(root.clone(), 0)]);
+ while let Some((element, depth)) = queue.pop_front() {
+ visited += 1;
+ if visited > 400 || cancellation::cancelled() {
+ return found;
+ }
+ let id = window_id(&element);
+ if !found.contains(&id) && checked.insert(id) {
+ if let Ok((owner, id, bounds)) = resolved_window(root, &element) {
+ if !contained || (owner != pid && inside(bounds, outer)) {
+ found.push(id);
+ }
+ }
+ }
+ if depth < MAX_DEPTH {
+ queue.extend(attr(&element, "AXChildren").map(list).unwrap_or_default().into_iter().map(|child| (child, depth + 1)));
+ }
+ }
+ }
+ found
+}
+
+fn prepare_destination<T: PartialEq>(
+ mut resolve: impl FnMut() -> Result<T, Failure>,
+ activate: impl FnOnce(&T) -> Result<(), Failure>,
+) -> Result<T, Failure> {
+ let destination = resolve()?;
+ activate(&destination)?;
+ if resolve()? != destination {
+ return Err(fail(ErrorCode::ActionFailed, "The keyboard destination changed while preparing input; observe again"));
+ }
+ Ok(destination)
+}
+
+fn keyboard_activation_window(host: &El, element: &El) -> Result<(i32, u32, Rect), Failure> {
+ let pid =
+ pid_of(host).filter(|pid| *pid > 0).ok_or_else(|| fail(ErrorCode::NoWindow, "The keyboard host is unavailable; observe again"))?;
+ let id = ancestor_window(
+ host,
+ element,
+ |element| attr(element, "AXParent"),
+ |ancestor| {
+ if !has_role(ancestor, &["AXWindow"]) {
+ return 0;
+ }
+ let id = window_id(ancestor);
+ if window_geometry(id).is_some_and(|(owner, _)| owner == pid) {
+ id
+ } else {
+ 0
+ }
+ },
+ )
+ .ok_or_else(|| fail(ErrorCode::ActionFailed, "The keyboard target is no longer linked to its host; observe again"))?;
+ let (owner, bounds) = window_geometry(id)
+ .filter(|(owner, _)| *owner == pid)
+ .ok_or_else(|| fail(ErrorCode::NoWindow, "The keyboard host window is unavailable; observe again"))?;
+ Ok((owner, id, bounds))
+}
+
+fn prepare_keyboard_host(host: &El, element: &El) -> Result<(), Failure> {
+ register_target(host, element)?;
+ prepare_destination(|| keyboard_activation_window(host, element), |(owner, window, _)| crate::focus::activate_target(*owner, *window))
+ .map(|_| ())
+}
+
+fn keyboard_destination(host: &El, element: &El) -> Result<i32, Failure> {
+ let (owner, _, _) = prepare_destination(
+ || resolved_window(host, element),
+ |(owner, _, _)| {
+ crate::intervention::include_target(*owner);
+ active()
+ },
+ )?;
+ Ok(owner)
+}
+
+fn prepare_keyboard(host: &El, element: &El) -> Result<i32, Failure> {
+ prepare_keyboard_host(host, element)?;
+ keyboard_destination(host, element)
+}
+
+pub(super) fn keyboard_target(ax: &El) -> Result<(i32, El), Failure> {
+ let activation_anchor = attr(ax, "AXFocusedUIElement")
+ .or_else(|| window_of(ax))
+ .ok_or_else(|| fail(ErrorCode::NoWindow, "Cannot identify the keyboard host window; observe again"))?;
+ prepare_keyboard_host(ax, &activation_anchor)?;
+ let focused = attr(ax, "AXFocusedUIElement");
+ let element = focused
+ .clone()
+ .or_else(|| window_of(ax))
+ .ok_or_else(|| fail(ErrorCode::NoWindow, "Cannot identify the keyboard target; observe again"))?;
+ let owner = keyboard_destination(ax, &element)?;
+ if attr(ax, "AXFocusedUIElement") != focused {
+ return Err(fail(ErrorCode::ActionFailed, "The focused element changed while preparing keyboard input; observe again"));
+ }
+ Ok((owner, ax.clone()))
+}
+
+pub(super) fn check_menu_action(
+ action: &Action,
+ mut menu_press: impl FnMut(u32) -> Result<bool, Failure>,
+ menu_open: impl FnOnce() -> Result<bool, Failure>,
+) -> Result<(), Failure> {
+ let guarded = match action {
+ Action::Click { target: Target::Index(index), button: Button::Left, count: 1, modifiers, hold_ms: 0 } if modifiers.is_empty() => {
+ !menu_press(*index)?
+ }
+ Action::Perform { index, action } if matches!(action.as_str(), "AXPress" | "Press") => !menu_press(*index)?,
+ Action::Click { .. } | Action::Scroll { .. } | Action::Drag { .. } => true,
+ _ => false,
+ };
+ if guarded && menu_open()? {
+ return Err(fail(ErrorCode::ActionFailed, MENU_OPEN));
+ }
+ Ok(())
+}
+
+const MAX_MENU_CHECKS: usize = 128;
+
+fn popup_menu_open<T>(
+ popups: &[T],
+ mut children: impl FnMut(&T) -> Result<Vec<T>, Failure>,
+ mut visible_menu: impl FnMut(&T) -> Result<bool, Failure>,
+) -> Result<bool, Failure> {
+ let mut checked = 0;
+ for popup in popups {
+ active()?;
+ checked += 1;
+ if checked > MAX_MENU_CHECKS {
+ return Err(menu_unavailable());
+ }
+ for child in children(popup)? {
+ active()?;
+ checked += 1;
+ if checked > MAX_MENU_CHECKS {
+ return Err(menu_unavailable());
+ }
+ if visible_menu(&child)? {
+ return Ok(true);
+ }
+ }
+ }
+ Ok(false)
+}
+
+fn menu_unavailable() -> Failure {
+ fail(ErrorCode::ActionFailed, "Popup menu inspection exceeded the 128-check limit; pointer input cannot be verified")
+}
+
+fn menu_attribute(element: &El, name: &str) -> Result<CFType, Failure> {
+ let mut value = std::ptr::null();
+ let status = unsafe { AXUIElementCopyAttributeValue(element.as_CFTypeRef(), cfs(name).as_concrete_TypeRef(), &mut value) };
+ ax_result(status, value).ok_or_else(|| {
+ fail(ErrorCode::ActionFailed, format!("Cannot read popup {name} (AXError {status}); pointer input cannot be verified"))
+ })
+}
+
+fn menu_attribute_type(name: &str, expected: &str) -> Failure {
+ fail(ErrorCode::ActionFailed, format!("Popup {name} is not {expected}; pointer input cannot be verified"))
+}
+
+pub(super) fn menu_open(ax: &El, popups: &[El]) -> Result<bool, Failure> {
+ active()?;
+ let kids = |e: &El| attr(e, "AXChildren").map(list).unwrap_or_default();
+ if kids(ax).iter().any(|c| has_role(c, &["AXMenu"]))
+ || attr(ax, "AXMenuBar").is_some_and(|bar| kids(&bar).iter().any(|item| bool_attr(item, "AXSelected") == Some(true)))
+ {
+ return Ok(true);
+ }
+ popup_menu_open(
+ popups,
+ |popup| {
+ let value = menu_attribute(popup, "AXChildren")?;
+ if value.downcast::<CFArray>().is_none() {
+ return Err(menu_attribute_type("AXChildren", "an array"));
+ }
+ Ok(list(value))
+ },
+ |child| {
+ let role = menu_attribute(child, "AXRole")?.downcast::<CFString>().ok_or_else(|| menu_attribute_type("AXRole", "a string"))?;
+ if role != "AXMenu" {
+ return Ok(false);
+ }
+ let hidden =
+ menu_attribute(child, "AXHidden")?.downcast::<CFBoolean>().ok_or_else(|| menu_attribute_type("AXHidden", "a boolean"))?;
+ Ok(!bool::from(hidden))
+ },
+ )
+}
+
+pub(super) const MENU_OPEN: &str =
+ "A menu is open, and pointer input inside or around it is unreliable. Click a menu item by its id, or press Escape to close the menu, then observe again.";
+
+pub(super) fn is_menu_item(e: &El) -> bool {
+ has_role(e, MENU_ITEMS)
+}
+
+fn in_web(e: &El) -> bool {
+ let mut cur = Some(e.clone());
+ for _ in 0..MAX_DEPTH {
+ let Some(c) = cur else { return false };
+ match str_attr(&c, "AXRole").as_deref() {
+ Some("AXWebArea") => return true,
+ Some("AXWindow") | Some("AXApplication") => return false,
+ _ => cur = attr(&c, "AXParent"),
+ }
+ }
+ false
+}
+
+pub(super) fn simulate(e: &El) -> bool {
+ if is_menu_item(e) {
+ return false;
+ }
+ if !actions(e).iter().any(|a| a == "AXPress") {
+ return true;
+ }
+ has_role(e, SELECTS) || in_web(e)
+}
+
+pub(super) fn scroll_area(e: &El) -> Option<El> {
+ let mut cur = Some(e.clone());
+ for _ in 0..MAX_DEPTH {
+ let c = cur?;
+ if has_role(&c, &["AXScrollArea"]) {
+ return Some(c);
+ }
+ if has_role(&c, &["AXWindow", "AXApplication"]) {
+ return None;
+ }
+ cur = attr(&c, "AXParent");
+ }
+ None
+}
+
+pub(super) fn area_at(ax: &El, p: CGPoint) -> Option<El> {
+ scroll_area(&element_at(ax, p)?)
+}
+
+fn number(e: &El, name: &str) -> Option<f64> {
+ attr(e, name)?.downcast::<CFNumber>()?.to_f64()
+}
+
+pub(super) fn scroll_pixels(amount: Amount, direction: Direction, viewport: Option<Rect>, fallback: Rect) -> i32 {
+ let view = viewport.unwrap_or(fallback);
+ match amount {
+ Amount::Pixels(n) => n as i32,
+ Amount::Pages(n) => {
+ let extent = if matches!(direction, Direction::Up | Direction::Down) { view.h } else { view.w };
+ (n.clamp(0.0, 20.0) * 0.85 * extent) as i32
+ }
+ }
+}
+
+pub(super) fn scroll_pages(host: &El, area: &El, direction: Direction, pages: f64) -> Result<bool, Failure> {
+ if pages.is_nan() || pages <= 0.0 {
+ return Ok(false);
+ }
+ let (action, bar, sign) = match direction {
+ Direction::Up => ("AXScrollUpByPage", "AXVerticalScrollBar", -1.0),
+ Direction::Down => ("AXScrollDownByPage", "AXVerticalScrollBar", 1.0),
+ Direction::Left => ("AXScrollLeftByPage", "AXHorizontalScrollBar", -1.0),
+ Direction::Right => ("AXScrollRightByPage", "AXHorizontalScrollBar", 1.0),
+ };
+ let whole = pages.fract() == 0.0 && pages <= 20.0;
+ if whole && actions(area).iter().any(|a| a == action) {
+ for _ in 0..pages as u32 {
+ perform(host, area, action)?;
+ }
+ return Ok(true);
+ }
+ let Some(bar) = attr(area, bar).filter(|b| settable(b, "AXValue")) else { return Ok(false) };
+ let (Some(value), Some(track)) = (number(&bar, "AXValue"), frame(&bar)) else { return Ok(false) };
+ let knob = attr(&bar, "AXChildren").map(list).unwrap_or_default().into_iter().find(|c| has_role(c, &["AXValueIndicator"]));
+ let Some(knob) = knob.and_then(|k| frame(&k)) else { return Ok(false) };
+ let vertical = matches!(direction, Direction::Up | Direction::Down);
+ let shown = if vertical { knob.h / track.h } else { knob.w / track.w };
+ if !(shown > 0.0 && shown < 1.0) {
+ return Ok(false);
+ }
+ let target = (value + sign * pages.min(20.0) * 0.85 * shown / (1.0 - shown)).clamp(0.0, 1.0);
+ if (target - value).abs() < 1e-6 {
+ return Ok(true);
+ }
+ register_target(host, &bar)?;
+ let status = set_attr(&bar, "AXValue", &CFNumber::from(target).as_CFType());
+ if status == 0 {
+ Ok(true)
+ } else {
+ Err(fail(ErrorCode::ActionFailed, format!("setting the scroll position failed (AXError {status})")))
+ }
+}
+
+fn date_value(before: &core_foundation::date::CFDate, value: &str, mask: u64, calendar: &objc2_foundation::NSCalendar) -> Option<CFType> {
+ use objc2_foundation::{NSCalendarUnit as Unit, NSDate, NSDateFormatter, NSDateFormatterStyle as Style};
+ if mask == 0 || mask & !0xee != 0 {
+ return None;
+ }
+ let formatter = NSDateFormatter::new();
+ formatter.setLenient(false);
+ formatter.setCalendar(Some(calendar));
+ let time_style = if mask & 2 != 0 { Style::MediumStyle } else { Style::ShortStyle };
+ match mask {
+ 0xe0 => formatter.setDateStyle(Style::ShortStyle),
+ 0xc | 0xe => formatter.setTimeStyle(time_style),
+ 0xc0 => formatter.setDateFormat(Some(&NSString::from_str("MMMM yyyy"))),
+ _ => {
+ formatter.setDateStyle(Style::ShortStyle);
+ formatter.setTimeStyle(time_style);
+ }
+ }
+ let parsed = formatter.dateFromString(&NSString::from_str(value))?;
+ let units = [
+ (0x80, Unit::Era),
+ (0x80, Unit::Year),
+ (0x40, Unit::Month),
+ (0x20, Unit::Day),
+ (8, Unit::Hour),
+ (4, Unit::Minute),
+ (2, Unit::Second),
+ ];
+ let all = units.iter().fold(Unit::Era | Unit::Nanosecond, |all, (_, unit)| all | *unit);
+ let original = NSDate::dateWithTimeIntervalSinceReferenceDate(before.abs_time());
+ let components = calendar.components_fromDate(all, &original);
+ for (bit, unit) in units {
+ if mask & bit != 0 {
+ components.setValue_forComponent(calendar.component_fromDate(unit, &parsed), unit);
+ }
+ }
+ let date = calendar.dateFromComponents(&components)?;
+ if units.iter().any(|(_, unit)| calendar.component_fromDate(*unit, &date) != components.valueForComponent(*unit)) {
+ return None;
+ }
+ Some(core_foundation::date::CFDate::new(date.timeIntervalSinceReferenceDate()).as_CFType())
+}
+
+fn has_focus(host: &El, element: &El) -> bool {
+ bool_attr(element, "AXFocused") == Some(true) || attr(host, "AXFocusedUIElement").as_ref() == Some(element)
+}
+
+pub(super) fn focus_field(host: &El, element: &El, click: impl FnOnce() -> Result<(), Failure>) -> Result<(), Failure> {
+ register_target(host, element)?;
+ if has_focus(host, element) {
+ return Ok(());
+ }
+ if settable(element, "AXFocused") {
+ let status = set_attr(element, "AXFocused", &CFBoolean::true_value().as_CFType());
+ if status != 0 {
+ return Err(fail(ErrorCode::ActionFailed, format!("Focusing the element failed (AXError {status})")));
+ }
+ return register_target(host, element).map(|_| ());
+ }
+ click()?;
+ if !has_focus(host, element) {
+ return Err(fail(ErrorCode::ActionFailed, "The element did not acquire keyboard focus; observe again"));
+ }
+ register_target(host, element)?;
+ Ok(())
+}
+
+pub(super) fn set_value(
+ pid: i32,
+ e: &El,
+ index: u32,
+ value: &str,
+ submit: Option<bool>,
+ focus: impl FnOnce() -> Result<(), Failure>,
+) -> Result<(), Failure> {
+ if !settable(e, "AXValue") {
+ return Err(fail(
+ ErrorCode::ActionFailed,
+ format!("element {index}'s value cannot be set; click it and use typeText or paste, or use its controls"),
+ ));
+ }
+ let before = attr(e, "AXValue");
+ let bad = |kind: &str| fail(ErrorCode::ActionFailed, format!("element {index} holds a {kind}, and {value:?} is not one"));
+ let new = match &before {
+ Some(v) if v.downcast::<CFBoolean>().is_some() => {
+ CFBoolean::from(text::flag(value).ok_or_else(|| bad("true/false value"))?).as_CFType()
+ }
+ Some(v) if v.downcast::<CFNumber>().is_some() => {
+ let n = text::number(value).ok_or_else(|| bad("number"))?;
+ if n.fract() == 0.0 && n.abs() < 9e15 {
+ CFNumber::from(n as i64).as_CFType()
+ } else {
+ CFNumber::from(n).as_CFType()
+ }
+ }
+ Some(v) if v.downcast::<core_foundation::date::CFDate>().is_some() => {
+ let mask = attr(e, "AXDateTimeComponents").and_then(|v| v.downcast::<CFNumber>()).and_then(|v| v.to_i64()).unwrap_or(0xee);
+ date_value(
+ &v.downcast::<core_foundation::date::CFDate>().unwrap(),
+ value,
+ mask as u64,
+ &objc2_foundation::NSCalendar::currentCalendar(),
+ )
+ .ok_or_else(|| bad("valid date/time in the system's format"))?
+ }
+ _ => cfs(value).as_CFType(),
+ };
+ let host = app_element(pid);
+ let texty = has_role(e, TEXT_ROLES);
+ let require_focus = || {
+ if has_focus(&host, e) {
+ Ok(())
+ } else {
+ Err(fail(ErrorCode::ActionFailed, format!("element {index} does not have keyboard focus; observe it before continuing")))
+ }
+ };
+ if texty {
+ focus()?;
+ } else {
+ register_target(&host, e)?;
+ }
+ active()?;
+ match set_attr(e, "AXValue", &new) {
+ 0 => {}
+ n => return Err(fail(ErrorCode::ActionFailed, format!("setting element {index}'s value failed (AXError {n})"))),
+ }
+ let old = before.as_ref().and_then(text_of);
+ let now = attr(e, "AXValue").as_ref().and_then(text_of);
+ if texty && now.as_deref() != Some(value) && now.is_some() && now == old {
+ return Err(fail(
+ ErrorCode::ActionFailed,
+ format!(
+ "element {index} did not take the value (it still reads {:?}); click it and use typeText or paste",
+ clip(&now.unwrap_or_default(), 80)
+ ),
+ ));
+ }
+ if submit == Some(true) {
+ if actions(e).iter().any(|a| a == "AXConfirm") {
+ perform(&host, e, "AXConfirm")?;
+ } else {
+ let target = prepare_keyboard(&host, e)?;
+ require_focus()?;
+ key(target, chord("return")?.0, CGEventFlags::empty(), None, 0)?;
+ }
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn keyboard_preparation_preserves_destination_and_propagates_detachment() {
+ let mut resolutions = [Ok((7, 81)), Ok((7, 81))].into_iter();
+ let mut activated = None;
+ assert_eq!(
+ prepare_destination(
+ || resolutions.next().unwrap(),
+ |destination| {
+ activated = Some(*destination);
+ Ok(())
+ }
+ )
+ .unwrap(),
+ (7, 81)
+ );
+ assert_eq!(activated, Some((7, 81)));
+ let mut resolutions = [Ok((7, 81)), Ok((9, 92))].into_iter();
+ assert!(prepare_destination(|| resolutions.next().unwrap(), |_| Ok(())).is_err());
+ let mut resolutions = [Ok((7, 81)), Err(fail(ErrorCode::NoWindow, "detached"))].into_iter();
+ assert!(prepare_destination(|| resolutions.next().unwrap(), |_| Ok(())).is_err());
+ assert!(prepare_destination::<(i32, u32)>(|| Err(fail(ErrorCode::NoWindow, "detached")), |_| panic!("unresolved target")).is_err());
+ }
+
+ #[test]
+ fn input_window_uses_nearest_ancestor_only_after_proving_host_link() {
+ let parent = |node: &u32| (*node > 0).then(|| node - 1);
+ assert_eq!(ancestor_window(&0, &3, parent, |node| if *node == 1 { 81 } else { 0 }), Some(81));
+ assert_eq!(ancestor_window(&0, &3, parent, |node| if *node == 3 { 92 } else { 81 }), Some(92));
+ assert_eq!(ancestor_window(&0, &3, parent, |_| 0), Some(0));
+ assert_eq!(ancestor_window(&0, &3, |_| None, |_| 92), None);
+ assert_eq!(ancestor_window(&0, &3, |_| Some(3), |_| 92), None);
+ assert_eq!(ancestor_window(&0, &(MAX_DEPTH as u32), parent, |_| 92), None);
+ let mut window_reads = Vec::new();
+ assert_eq!(
+ ancestor_window(&0, &3, parent, |node| {
+ window_reads.push(*node);
+ 92
+ }),
+ Some(92)
+ );
+ assert_eq!(window_reads, vec![3]);
+ }
+
+ #[test]
+ fn input_window_revalidation_tracks_original_anchor_reparenting() {
+ let resolve = |ancestor| {
+ ancestor_window(
+ &0,
+ &3,
+ |node| match *node {
+ 3 => ancestor,
+ 1 | 2 => Some(0),
+ _ => None,
+ },
+ |node| match *node {
+ 1 => 81,
+ 2 => 92,
+ _ => 0,
+ },
+ )
+ };
+ assert_eq!(resolve(Some(1)), Some(81));
+ assert_eq!(resolve(Some(2)), Some(92));
+ assert_eq!(resolve(None), None);
+ }
+
+ #[test]
+ fn menu_preflight_covers_press_aliases_and_pointer_variants() {
+ let click = Action::Click { target: Target::Index(3), button: Button::Left, count: 1, modifiers: vec![], hold_ms: 0 };
+ assert!(check_menu_action(&click, |_| Ok(true), || panic!("safe menu press")).is_ok());
+ assert!(check_menu_action(&click, |_| Ok(false), || Ok(true)).is_err());
+ for name in ["Press", "AXPress"] {
+ let action = Action::Perform { index: 3, action: name.into() };
+ assert!(check_menu_action(&action, |_| Ok(false), || Ok(true)).is_err());
+ assert!(check_menu_action(&action, |_| Ok(true), || panic!("safe menu press")).is_ok());
+ }
+ for action in [
+ Action::Click { button: Button::Right, target: Target::Index(3), count: 1, modifiers: vec![], hold_ms: 0 },
+ Action::Click { button: Button::Left, target: Target::Index(3), count: 2, modifiers: vec![], hold_ms: 0 },
+ Action::Scroll { target: Some(Target::Index(3)), direction: Direction::Down, amount: Amount::Pixels(10) },
+ Action::Drag { from: Point { x: 1.0, y: 1.0 }, to: Point { x: 2.0, y: 2.0 } },
+ ] {
+ assert!(check_menu_action(&action, |_| panic!("pointer route"), || Ok(true)).is_err());
+ assert!(check_menu_action(&action, |_| panic!("pointer route"), || Ok(false)).is_ok());
+ }
+ for action in [Action::Perform { index: 3, action: "AXCancel".into() }, Action::PressKey { key: "ESC".into(), hold_ms: 0 }] {
+ assert!(check_menu_action(&action, |_| panic!("nonpointer action"), || panic!("nonpointer action")).is_ok());
+ }
+ assert!(check_menu_action(&click, |_| Ok(false), || Err(menu_unavailable())).is_err());
+ }
+
+ #[test]
+ fn popup_candidates_use_current_children_and_refuse_unknown_state() {
+ assert!(super::popup_menu_open(&[1], |_| Ok(vec![2]), |child| Ok(*child == 2)).unwrap());
+ assert!(!super::popup_menu_open(&[1], |_| Ok(vec![]), |_| panic!("no children")).unwrap());
+ assert!(!super::popup_menu_open(&[1], |_| Ok(vec![2]), |_| Ok(false)).unwrap());
+ assert!(super::popup_menu_open(&[1], |_| Err(super::menu_unavailable()), |_| Ok(false)).is_err());
+ assert!(super::popup_menu_open(&[1], |_| Ok(vec![2]), |_| Err(super::menu_unavailable())).is_err());
+ assert!(super::popup_menu_open(&[1], |_| Ok(vec![2; super::MAX_MENU_CHECKS]), |_| Ok(false)).is_err());
+ }
+
+ use super::*;
+
+ #[test]
+ fn date_values_preserve_components_outside_the_control() {
+ use core_foundation::date::CFDate;
+ use objc2_foundation::{NSCalendar, NSCalendarUnit as Unit, NSDate, NSDateFormatter, NSDateFormatterStyle as Style};
+ let calendar = NSCalendar::currentCalendar();
+ let original = NSDate::dateWithTimeIntervalSinceReferenceDate(737_985_757.0);
+ let target = NSDate::dateWithTimeIntervalSinceReferenceDate(738_165_123.0);
+ let before = CFDate::new(original.timeIntervalSinceReferenceDate());
+ for (mask, changed, preserved) in [
+ (0xe0, vec![Unit::Year, Unit::Month, Unit::Day], vec![Unit::Hour, Unit::Minute, Unit::Second]),
+ (0xc, vec![Unit::Hour, Unit::Minute], vec![Unit::Year, Unit::Month, Unit::Day, Unit::Second]),
+ (0xe, vec![Unit::Hour, Unit::Minute, Unit::Second], vec![Unit::Year, Unit::Month, Unit::Day]),
+ (0xee, vec![Unit::Year, Unit::Month, Unit::Day, Unit::Hour, Unit::Minute, Unit::Second], vec![]),
+ ] {
+ let formatter = NSDateFormatter::new();
+ if mask & 0xe0 != 0 {
+ formatter.setDateStyle(Style::ShortStyle);
+ }
+ if mask & 0xe != 0 {
+ formatter.setTimeStyle(if mask & 2 != 0 { Style::MediumStyle } else { Style::ShortStyle });
+ }
+ let value = formatter.stringFromDate(&target).to_string();
+ let result = date_value(&before, &value, mask, &calendar).unwrap().downcast::<CFDate>().unwrap();
+ let result = NSDate::dateWithTimeIntervalSinceReferenceDate(result.abs_time());
+ for unit in changed {
+ assert_eq!(calendar.component_fromDate(unit, &result), calendar.component_fromDate(unit, &target));
+ }
+ for unit in preserved {
+ assert_eq!(calendar.component_fromDate(unit, &result), calendar.component_fromDate(unit, &original));
+ }
+ }
+ assert!(date_value(&before, "not a date", 0xe0, &calendar).is_none());
+ assert!(date_value(&before, "not a time", 0xc, &calendar).is_none());
+ }
+
+ #[test]
+ fn date_values_update_era_with_year() {
+ use core_foundation::date::CFDate;
+ use objc2_foundation::{
+ NSCalendar, NSCalendarIdentifierJapanese, NSCalendarUnit as Unit, NSDate, NSDateFormatter, NSDateFormatterStyle,
+ };
+ let calendar = NSCalendar::calendarWithIdentifier(unsafe { NSCalendarIdentifierJapanese }).unwrap();
+ let original = NSDate::dateWithTimeIntervalSinceReferenceDate(550_000_000.0);
+ let target = NSDate::dateWithTimeIntervalSinceReferenceDate(738_165_123.0);
+ assert_ne!(calendar.component_fromDate(Unit::Era, &original), calendar.component_fromDate(Unit::Era, &target));
+ let formatter = NSDateFormatter::new();
+ formatter.setCalendar(Some(&calendar));
+ formatter.setDateStyle(NSDateFormatterStyle::ShortStyle);
+ let value = formatter.stringFromDate(&target).to_string();
+ let result = date_value(&CFDate::new(original.timeIntervalSinceReferenceDate()), &value, 0xe0, &calendar)
+ .unwrap()
+ .downcast::<CFDate>()
+ .unwrap();
+ let result = NSDate::dateWithTimeIntervalSinceReferenceDate(result.abs_time());
+ for unit in [Unit::Era, Unit::Year, Unit::Month, Unit::Day] {
+ assert_eq!(calendar.component_fromDate(unit, &result), calendar.component_fromDate(unit, &target));
+ }
+ }
+
+ #[test]
+ fn date_values_reject_normalization_of_hidden_day() {
+ use core_foundation::date::CFDate;
+ use objc2_foundation::{NSCalendar, NSCalendarUnit as Unit, NSDate, NSDateFormatter};
+ let calendar = NSCalendar::currentCalendar();
+ let components =
+ calendar.components_fromDate(Unit::Year | Unit::Month | Unit::Day, &NSDate::dateWithTimeIntervalSinceReferenceDate(0.0));
+ components.setValue_forComponent(2024, Unit::Year);
+ components.setValue_forComponent(1, Unit::Month);
+ components.setValue_forComponent(31, Unit::Day);
+ let original = calendar.dateFromComponents(&components).unwrap();
+ components.setValue_forComponent(2, Unit::Month);
+ components.setValue_forComponent(1, Unit::Day);
+ let target = calendar.dateFromComponents(&components).unwrap();
+ let formatter = NSDateFormatter::new();
+ formatter.setDateFormat(Some(&NSString::from_str("MMMM yyyy")));
+ let value = formatter.stringFromDate(&target).to_string();
+ assert!(date_value(&CFDate::new(original.timeIntervalSinceReferenceDate()), &value, 0xc0, &calendar).is_none());
+ }
+
+ #[test]
+ fn index_click_requires_a_positive_target_hit() {
+ let rect = Rect { x: 10.0, y: 20.0, w: 100.0, h: 80.0 };
+ for hits in [vec![None; 5], vec![Some("ancestor"); 5], vec![None, Some("cover"), Some("ancestor"), None, Some("cover")]] {
+ let mut hits = hits.into_iter();
+ assert!(clickable_point(rect, |_| hits.next().flatten(), |hit| *hit == "target" || *hit == "descendant").is_err());
+ }
+ let mut hits = [Some("cover"), None, Some("descendant")].into_iter();
+ let descendant = clickable_point(rect, |_| hits.next().flatten(), |hit| *hit == "target" || *hit == "descendant").unwrap();
+ assert_eq!((descendant.x, descendant.y), (85.0, 60.0));
+ let target = clickable_point(rect, |_| Some("target"), |hit| *hit == "target").unwrap();
+ assert_eq!((target.x, target.y), (60.0, 60.0));
+ }
+
+ #[test]
+ fn page_wheel_fallback_uses_the_scroll_pane_viewport() {
+ let pane = Rect { x: 0.0, y: 0.0, w: 200.0, h: 400.0 };
+ let row = Rect { x: 0.0, y: 0.0, w: 150.0, h: 20.0 };
+ let window = Rect { x: 0.0, y: 0.0, w: 800.0, h: 1000.0 };
+ assert_eq!(scroll_pixels(Amount::Pages(1.0), Direction::Down, Some(pane), row), 340);
+ assert_eq!(scroll_pixels(Amount::Pages(0.5), Direction::Right, Some(pane), window), 85);
+ assert_eq!(scroll_pixels(Amount::Pages(0.5), Direction::Up, None, window), 425);
+ assert_eq!(scroll_pixels(Amount::Pixels(17), Direction::Down, Some(pane), row), 17);
+ }
+}
diff --git a/desktop/desk/src/mac/markers.rs b/desktop/desk/src/mac/markers.rs
new file mode 100644
index 000000000..7762725fd
--- /dev/null
+++ b/desktop/desk/src/mac/markers.rs
@@ -0,0 +1,155 @@
+use super::*;
+
+#[link(name = "ApplicationServices", kind = "framework")]
+extern "C" {
+ fn AXTextMarkerRangeGetTypeID() -> usize;
+}
+
+pub(super) enum Read {
+ Text { range: El, raw: String, attributed: CFAttributedString },
+ Omitted(usize),
+}
+
+fn parameter(e: &El, name: &str, argument: &El) -> Option<CFType> {
+ if cancellation::cancelled() {
+ return None;
+ }
+ let mut value = std::ptr::null();
+ let status = unsafe {
+ AXUIElementCopyParameterizedAttributeValue(e.as_CFTypeRef(), cfs(name).as_concrete_TypeRef(), argument.as_CFTypeRef(), &mut value)
+ };
+ ax_result(status, value)
+}
+
+fn bounded(range: El, mut query: impl FnMut(&str, &El) -> Option<CFType>) -> Option<Read> {
+ if range.type_of() != unsafe { AXTextMarkerRangeGetTypeID() } || cancellation::cancelled() {
+ return None;
+ }
+ let length = query("AXLengthForTextMarkerRange", &range)?.downcast::<CFNumber>()?.to_i64()?;
+ let length = usize::try_from(length).ok().filter(|length| *length > 0)?;
+ if length > tree::MAX_TEXT {
+ return Some(Read::Omitted(length));
+ }
+ if cancellation::cancelled() {
+ return None;
+ }
+ let attributed = query("AXAttributedStringForTextMarkerRange", &range)?.downcast::<CFAttributedString>()?;
+ if usize::try_from(attributed.char_len()).ok()? != length {
+ return None;
+ }
+ let raw = unsafe { CFString::wrap_under_get_rule(CFAttributedStringGetString(attributed.as_concrete_TypeRef())) }.to_string();
+ Some(Read::Text { range, raw, attributed })
+}
+
+fn read_element(e: &El, mut query: impl FnMut(&str, &El) -> Option<CFType>) -> Option<Read> {
+ if cancellation::cancelled() {
+ return None;
+ }
+ let range = query("AXTextMarkerRangeForUIElement", e)?;
+ let result = bounded(range.clone(), &mut query)?;
+ if cancellation::cancelled() || query("AXTextMarkerRangeForUIElement", e).as_ref() != Some(&range) {
+ return None;
+ }
+ Some(result)
+}
+
+pub(super) fn read(e: &El) -> Option<Read> {
+ read_element(e, |name, argument| parameter(e, name, argument))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ extern "C" {
+ fn AXTextMarkerRangeCreate(allocator: CFTypeRef, start: CFTypeRef, end: CFTypeRef) -> CFTypeRef;
+ fn AXTextMarkerCreate(allocator: CFTypeRef, bytes: *const u8, length: isize) -> CFTypeRef;
+ }
+
+ fn range() -> El {
+ range_ending_at(b"b")
+ }
+
+ fn range_ending_at(bytes: &[u8]) -> El {
+ let start = unsafe { CFType::wrap_under_create_rule(AXTextMarkerCreate(std::ptr::null(), b"a".as_ptr(), 1)) };
+ let end = unsafe { CFType::wrap_under_create_rule(AXTextMarkerCreate(std::ptr::null(), bytes.as_ptr(), bytes.len() as isize)) };
+ unsafe { CFType::wrap_under_create_rule(AXTextMarkerRangeCreate(std::ptr::null(), start.as_CFTypeRef(), end.as_CFTypeRef())) }
+ }
+
+ #[test]
+ fn marker_observation_uses_exact_element_range_and_rechecks_it() {
+ let element = cfs("leaf").as_CFType();
+ let expected_range = range();
+ assert_eq!(expected_range, range());
+ let mut calls = vec![];
+ let result = read_element(&element, |name, argument| {
+ calls.push(name.to_string());
+ match name {
+ "AXTextMarkerRangeForUIElement" => {
+ assert_eq!(argument, &element);
+ Some(range())
+ }
+ "AXLengthForTextMarkerRange" => {
+ assert_eq!(argument, &expected_range);
+ Some(CFNumber::from(4).as_CFType())
+ }
+ "AXAttributedStringForTextMarkerRange" => {
+ assert_eq!(argument, &expected_range);
+ Some(CFAttributedString::new(&cfs("leaf")).as_CFType())
+ }
+ _ => panic!("unexpected range query"),
+ }
+ });
+ assert!(matches!(result, Some(Read::Text { raw, .. }) if raw == "leaf"));
+ assert_eq!(
+ calls,
+ [
+ "AXTextMarkerRangeForUIElement",
+ "AXLengthForTextMarkerRange",
+ "AXAttributedStringForTextMarkerRange",
+ "AXTextMarkerRangeForUIElement"
+ ]
+ );
+ let mut range_reads = 0;
+ assert!(read_element(&element, |name, _| match name {
+ "AXTextMarkerRangeForUIElement" => {
+ range_reads += 1;
+ Some(if range_reads == 1 { range() } else { range_ending_at(b"c") })
+ }
+ "AXLengthForTextMarkerRange" => Some(CFNumber::from(4).as_CFType()),
+ _ => Some(CFAttributedString::new(&cfs("leaf")).as_CFType()),
+ })
+ .is_none());
+ }
+
+ #[test]
+ fn marker_observation_never_reads_unbounded_ranges() {
+ for length in [-1, 0, 1001, i64::MAX] {
+ let mut calls = vec![];
+ let result = bounded(range(), |name, _| {
+ calls.push(name.to_string());
+ Some(CFNumber::from(length).as_CFType())
+ });
+ assert_eq!(calls, ["AXLengthForTextMarkerRange"]);
+ assert_eq!(matches!(result, Some(Read::Omitted(_))), length > 1000);
+ }
+ }
+
+ #[test]
+ fn marker_observation_retains_exact_text_and_opaque_range() {
+ let range = range();
+ let result = bounded(range.clone(), |name, _| match name {
+ "AXLengthForTextMarkerRange" => Some(CFNumber::from(5).as_CFType()),
+ "AXAttributedStringForTextMarkerRange" => Some(CFAttributedString::new(&cfs(" 😀x ")).as_CFType()),
+ _ => None,
+ });
+ let Some(Read::Text { range: retained, raw, .. }) = result else { panic!() };
+ assert_eq!(retained, range);
+ assert_eq!(raw, " 😀x ");
+ assert!(bounded(range, |name, _| match name {
+ "AXLengthForTextMarkerRange" => Some(CFNumber::from(4).as_CFType()),
+ _ => Some(CFAttributedString::new(&cfs("changed")).as_CFType()),
+ })
+ .is_none());
+ }
+}
diff --git a/desktop/desk/src/main.rs b/desktop/desk/src/main.rs
index c4d698e5c..6c25af689 100644
--- a/desktop/desk/src/main.rs
+++ b/desktop/desk/src/main.rs
@@ -1,5 +1,21 @@
mod cancellation;
+#[cfg(target_os = "macos")]
+mod focus;
+#[cfg(any(target_os = "macos", test))]
+mod focus_protocol;
+#[cfg(target_os = "macos")]
+mod intervention;
+#[cfg(any(target_os = "macos", test))]
+mod keys;
mod protocol;
+mod text;
+#[cfg(any(target_os = "macos", test))]
+mod tree;
+
+#[cfg(target_os = "macos")]
+mod capture;
+#[cfg(target_os = "macos")]
+mod pasteboard;
#[cfg(target_os = "macos")]
mod mac;
@@ -102,9 +118,40 @@ fn print(reply: Reply, png: Option<&str>) -> Result<(), String> {
}
}
+struct InputLines(std::sync::mpsc::Receiver<std::io::Result<String>>);
+
+impl InputLines {
+ fn new() -> Self {
+ let (sender, receiver) = std::sync::mpsc::sync_channel(1);
+ std::thread::spawn(move || {
+ for line in std::io::stdin().lock().lines() {
+ if sender.send(line).is_err() {
+ break;
+ }
+ }
+ });
+ Self(receiver)
+ }
+}
+
+impl Iterator for InputLines {
+ type Item = std::io::Result<String>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ while !cancellation::cancelled() {
+ match self.0.recv_timeout(std::time::Duration::from_millis(20)) {
+ Ok(line) => return Some(line),
+ Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
+ Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => return None,
+ }
+ }
+ None
+ }
+}
+
fn serve(desk: &mut mac::Desk) -> Result<(), String> {
let mut out = std::io::stdout().lock();
- let mut lines = std::io::stdin().lock().lines();
+ let mut lines = InputLines::new();
while let Some(line) = lines.next() {
if cancellation::cancelled() {
break;
diff --git a/desktop/desk/src/pasteboard.rs b/desktop/desk/src/pasteboard.rs
new file mode 100644
index 000000000..c6355db6e
--- /dev/null
+++ b/desktop/desk/src/pasteboard.rs
@@ -0,0 +1,193 @@
+use crate::protocol::{ErrorCode, Failure, PasteFormat};
+use crate::text;
+use objc2::rc::Retained;
+use objc2::runtime::ProtocolObject;
+use objc2_app_kit::{NSPasteboard, NSPasteboardItem, NSPasteboardTypeHTML, NSPasteboardTypeString, NSPasteboardWriting};
+use objc2_foundation::{NSArray, NSData, NSString};
+use std::time::{Duration, Instant};
+
+const MARKERS: &[&str] = &["org.nspasteboard.TransientType", "org.nspasteboard.AutoGeneratedType"];
+const CONCEALED: &[&str] = &["org.nspasteboard.ConcealedType"];
+const SHOW_TIMEOUT: Duration = Duration::from_secs(3);
+
+type Saved = Vec<Vec<(Retained<NSString>, Retained<NSData>)>>;
+
+fn fail(code: ErrorCode, m: &str) -> Failure {
+ Failure::new(code, m)
+}
+
+fn snapshot(board: &NSPasteboard) -> Result<(Saved, isize), Failure> {
+ let before = board.changeCount();
+ let mut saved = vec![];
+ for item in board.pasteboardItems().iter().flat_map(|items| items.iter()) {
+ let mut types = vec![];
+ for t in item.types() {
+ if CONCEALED.contains(&t.to_string().as_str()) {
+ return Err(fail(
+ ErrorCode::ActionFailed,
+ "The clipboard holds a concealed item, such as a password, so desk did not paste. Use typeText instead.",
+ ));
+ }
+ let d = item
+ .dataForType(&t)
+ .ok_or_else(|| fail(ErrorCode::ActionFailed, "Could not preserve the current clipboard; paste was not attempted."))?;
+ types.push((t, d));
+ }
+ saved.push(types);
+ }
+ if board.changeCount() != before {
+ return Err(fail(ErrorCode::PasteConflict, "The clipboard changed while being saved; paste was not attempted."));
+ }
+ Ok((saved, before))
+}
+
+fn write(board: &NSPasteboard, items: Vec<Retained<NSPasteboardItem>>, expected: isize) -> Result<(bool, isize), Failure> {
+ let objects: Vec<Retained<ProtocolObject<dyn NSPasteboardWriting>>> = items.into_iter().map(ProtocolObject::from_retained).collect();
+ let objects = NSArray::from_retained_slice(&objects);
+ if board.changeCount() != expected {
+ return Err(fail(ErrorCode::PasteConflict, CONFLICT));
+ }
+ let cleared = board.clearContents();
+ if board.changeCount() != cleared {
+ return Err(fail(ErrorCode::PasteConflict, CONFLICT));
+ }
+ Ok((objects.is_empty() || board.writeObjects(&objects), cleared))
+}
+
+fn restore(board: &NSPasteboard, saved: &Saved, expected: isize) -> bool {
+ let mut items = Vec::with_capacity(saved.len());
+ for types in saved {
+ let item = NSPasteboardItem::new();
+ for (kind, data) in types {
+ if !item.setData_forType(data, kind) {
+ return false;
+ }
+ }
+ items.push(item);
+ }
+ write(board, items, expected).is_ok_and(|(written, _)| written)
+}
+
+pub enum Check {
+ Landed,
+ NotYet,
+ Unreadable,
+}
+
+const CONFLICT: &str = "The clipboard changed during the paste, probably because the user copied something. \
+The app may have pasted that instead of your text. Observe the app and check what was pasted before continuing.";
+
+fn uncertain(board: &NSPasteboard, ours: isize, why: &str) -> Failure {
+ if board.changeCount() != ours {
+ return fail(ErrorCode::PasteConflict, &format!("{why}. {CONFLICT}"));
+ }
+ fail(
+ ErrorCode::ActionFailed,
+ &format!(
+ "desk could not confirm that the app pasted the text: {why}. The pasted text is still on the clipboard, and the user's previous clipboard was not restored, \
+because restoring it now could let the app paste that instead. Observe the field to check what was pasted before continuing."
+ ),
+ )
+}
+
+pub fn paste(
+ text: &str,
+ format: PasteFormat,
+ keystroke: impl FnOnce() -> Result<(), Failure>,
+ check: &mut dyn FnMut() -> Check,
+ interrupted: &dyn Fn() -> Result<(), Failure>,
+) -> Result<(), Failure> {
+ let (plain, html) = text::payload(text, format);
+ let mut data = vec![(unsafe { NSPasteboardTypeString }.to_string(), plain.into_bytes())];
+ if let Some(html) = html {
+ data.push((unsafe { NSPasteboardTypeHTML }.to_string(), html.into_bytes()));
+ }
+ data.extend(MARKERS.iter().map(|marker| (marker.to_string(), vec![])));
+ let item = NSPasteboardItem::new();
+ for (kind, bytes) in data {
+ if !item.setData_forType(&NSData::with_bytes(&bytes), &NSString::from_str(&kind)) {
+ return Err(fail(ErrorCode::ActionFailed, "Could not put the text on the clipboard."));
+ }
+ }
+
+ let board = NSPasteboard::generalPasteboard();
+ let (saved, previous) = snapshot(&board)?;
+ interrupted()?;
+ let (written, cleared) = write(&board, vec![item], previous)?;
+ if !written {
+ return Err(fail(
+ ErrorCode::ActionFailed,
+ if restore(&board, &saved, cleared) {
+ "Could not put the text on the clipboard."
+ } else {
+ "Could not put the text on the clipboard, and the previous clipboard could not be restored."
+ },
+ ));
+ }
+ let ours = board.changeCount();
+ if let Err(e) = keystroke() {
+ return Err(uncertain(&board, ours, &format!("sending Cmd+V failed ({})", e.message)));
+ }
+
+ let start = Instant::now();
+ loop {
+ crate::cancellation::wait(Duration::from_millis(100));
+ if board.changeCount() != ours {
+ return Err(fail(ErrorCode::PasteConflict, CONFLICT));
+ }
+ if let Err(e) = interrupted() {
+ return Err(uncertain(&board, ours, &format!("the paste was interrupted after Cmd+V was sent ({})", e.message)));
+ }
+ match check() {
+ Check::Landed => break,
+ Check::Unreadable => return Err(uncertain(&board, ours, "desk cannot read the text of the app's focused field")),
+ Check::NotYet if start.elapsed() >= SHOW_TIMEOUT => {
+ return Err(uncertain(&board, ours, "the focused field does not show the pasted text"));
+ }
+ Check::NotYet => {}
+ }
+ }
+ if !restore(&board, &saved, ours) {
+ return Err(fail(
+ ErrorCode::ActionFailed,
+ "The text was pasted, but the previous clipboard could not be restored. Check the clipboard before continuing.",
+ ));
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn item(types: &[(&str, &[u8])]) -> Retained<NSPasteboardItem> {
+ let item = NSPasteboardItem::new();
+ for (kind, bytes) in types {
+ assert!(item.setData_forType(&NSData::with_bytes(bytes), &NSString::from_str(kind)));
+ }
+ item
+ }
+
+ #[test]
+ fn restores_every_item_and_type_without_overwriting_a_new_owner() {
+ let board = NSPasteboard::pasteboardWithUniqueName();
+ let original = vec![
+ item(&[("public.utf8-plain-text", b"saved"), ("test.qm.binary", &[0, 255, 4])]),
+ item(&[("public.html", b"<b>second</b>")]),
+ ];
+ assert!(write(&board, original, board.changeCount()).unwrap().0);
+ let (saved, previous) = snapshot(&board).unwrap();
+ assert!(write(&board, vec![item(&[("public.utf8-plain-text", b"temporary")])], previous).unwrap().0);
+ assert!(restore(&board, &saved, board.changeCount()));
+ let items = board.pasteboardItems().unwrap();
+ assert_eq!(items.len(), 2);
+ assert_eq!(items.objectAtIndex(0).dataForType(&NSString::from_str("test.qm.binary")).unwrap().to_vec(), vec![0, 255, 4]);
+ assert_eq!(items.objectAtIndex(0).dataForType(&NSString::from_str("public.utf8-plain-text")).unwrap().to_vec(), b"saved");
+ assert_eq!(items.objectAtIndex(1).dataForType(&NSString::from_str("public.html")).unwrap().to_vec(), b"<b>second</b>");
+ let before_copy = board.changeCount();
+ assert!(write(&board, vec![item(&[("public.utf8-plain-text", b"new owner")])], before_copy).unwrap().0);
+ assert!(!restore(&board, &saved, before_copy));
+ assert_eq!(board.dataForType(&NSString::from_str("public.utf8-plain-text")).unwrap().to_vec(), b"new owner");
+ board.clearContents();
+ }
+}
diff --git a/desktop/desk/src/protocol.rs b/desktop/desk/src/protocol.rs
index b62da11e2..0c706ad38 100644
--- a/desktop/desk/src/protocol.rs
+++ b/desktop/desk/src/protocol.rs
@@ -62,6 +62,11 @@ pub enum Action {
TypeText {
text: String,
},
+ Paste {
+ text: String,
+ #[serde(default)]
+ format: PasteFormat,
+ },
PressKey {
key: String,
#[serde(default)]
@@ -91,6 +96,8 @@ pub enum Action {
SetValue {
index: u32,
value: String,
+ #[serde(default)]
+ submit: Option<bool>,
},
Perform {
index: u32,
@@ -147,6 +154,15 @@ impl Default for Amount {
}
}
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum PasteFormat {
+ #[default]
+ Text,
+ Markdown,
+ Html,
+}
+
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Selection {
@@ -199,6 +215,7 @@ pub struct Observation {
pub bundle_id: Option<String>,
pub pid: i32,
pub window: String,
+ pub window_id: u32,
pub frame: Rect,
pub elements: u32,
pub truncated: bool,
@@ -208,7 +225,7 @@ pub struct Observation {
pub screenshot: Option<Screenshot>,
}
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Rect {
pub x: f64,
@@ -224,6 +241,7 @@ pub struct Screenshot {
pub width: u32,
pub height: u32,
pub pixels_per_point: f64,
+ pub frame: Rect,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -245,6 +263,7 @@ pub enum ErrorCode {
BadIndex,
UnknownKey,
ActionFailed,
+ PasteConflict,
}
impl Failure {
diff --git a/desktop/desk/src/text.rs b/desktop/desk/src/text.rs
new file mode 100644
index 000000000..d6f52e3bd
--- /dev/null
+++ b/desktop/desk/src/text.rs
@@ -0,0 +1,620 @@
+use crate::protocol::{PasteFormat, Selection};
+use sha2::{Digest, Sha256};
+use std::collections::HashMap;
+
+const KEYPAD: &[u16] = &[65, 67, 69, 71, 75, 76, 78, 81, 82, 83, 84, 85, 86, 87, 88, 89, 91, 92];
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Stroke {
+ pub code: u16,
+ pub shift: bool,
+ pub option: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Key {
+ Return,
+ Tab,
+ Mapped(char, Stroke),
+ Unicode(char),
+}
+
+pub fn layout(translate: impl Fn(u16, bool, bool) -> Option<char>) -> HashMap<char, Stroke> {
+ let mut map = HashMap::new();
+ for (shift, option) in [(false, false), (true, false), (false, true), (true, true)] {
+ for code in 0..128u16 {
+ if KEYPAD.contains(&code) {
+ continue;
+ }
+ if let Some(c) = translate(code, shift, option).filter(|c| !c.is_control() && !('\u{f700}'..='\u{f8ff}').contains(c)) {
+ map.entry(c).or_insert(Stroke { code, shift, option });
+ }
+ }
+ }
+ map
+}
+
+pub fn plan(text: &str, layout: &HashMap<char, Stroke>) -> Vec<Key> {
+ let mut keys = vec![];
+ let mut chars = text.chars().peekable();
+ while let Some(c) = chars.next() {
+ keys.push(match c {
+ '\r' => {
+ chars.next_if_eq(&'\n');
+ Key::Return
+ }
+ '\n' => Key::Return,
+ '\t' => Key::Tab,
+ c => layout.get(&c).map_or(Key::Unicode(c), |s| Key::Mapped(c, *s)),
+ });
+ }
+ keys
+}
+
+pub fn payload(text: &str, format: PasteFormat) -> (String, Option<String>) {
+ match format {
+ PasteFormat::Text => (text.to_string(), None),
+ PasteFormat::Markdown => {
+ let mut html = String::from("<meta charset=\"utf-8\">");
+ pulldown_cmark::html::push_html(&mut html, pulldown_cmark::Parser::new_ext(text, pulldown_cmark::Options::all()));
+ (text.to_string(), Some(html))
+ }
+ PasteFormat::Html => (strip_html(text), Some(format!("<meta charset=\"utf-8\">{text}"))),
+ }
+}
+
+pub fn flag(s: &str) -> Option<bool> {
+ match s.trim().to_ascii_lowercase().as_str() {
+ "true" | "yes" | "on" | "1" | "checked" => Some(true),
+ "false" | "no" | "off" | "0" | "unchecked" => Some(false),
+ _ => None,
+ }
+}
+
+pub fn number(s: &str) -> Option<f64> {
+ s.trim().parse::<f64>().ok().filter(|n| n.is_finite()).or_else(|| flag(s).map(|b| if b { 1.0 } else { 0.0 }))
+}
+
+pub fn expected(text: &str, format: PasteFormat) -> Vec<String> {
+ let (plain, html) = payload(text, format);
+ let rendered = html.map(|h| strip_html(&h)).filter(|r| *r != plain);
+ std::iter::once(plain).chain(rendered).collect()
+}
+
+fn norm(s: &[u16]) -> String {
+ String::from_utf16_lossy(s).replace("\r\n", "\n").replace(['\r', '\u{2028}', '\u{2029}'], "\n").replace('\u{a0}', " ")
+}
+
+fn insertion(before: &str, pasted: &str, after: &str) -> bool {
+ if after.len() != before.len() + pasted.len() {
+ return false;
+ }
+ let prefix = before.bytes().zip(after.bytes()).take_while(|(x, y)| x == y).count();
+ let suffix = before.bytes().rev().zip(after.bytes().rev()).take_while(|(x, y)| x == y).count();
+ let mut start = before.len() - suffix;
+ while start <= prefix && !after.is_char_boundary(start) {
+ start += 1;
+ }
+ start <= prefix && after[start..].find(pasted).is_some_and(|offset| start + offset <= prefix)
+}
+
+pub fn inserted(before: &str, selection: Option<(usize, usize)>, after: &str, pasted: &str) -> bool {
+ let b: Vec<u16> = before.encode_utf16().collect();
+ let after = norm(&after.encode_utf16().collect::<Vec<_>>());
+ let pasted = norm(&pasted.encode_utf16().collect::<Vec<_>>());
+ if pasted.is_empty() || norm(&b) == after {
+ return false;
+ }
+ match selection {
+ Some((start, len)) if start + len <= b.len() => format!("{}{pasted}{}", norm(&b[..start]), norm(&b[start + len..])) == after,
+ Some(_) => false,
+ None => insertion(&norm(&b), &pasted, &after),
+ }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct VisibleText {
+ pub text: String,
+ pub start: usize,
+ pub end: usize,
+ pub shown_end: usize,
+ pub total: usize,
+}
+
+pub fn visible_text(value: &str, range: [isize; 2], limit: usize) -> Option<VisibleText> {
+ let start = usize::try_from(range[0]).ok()?;
+ let end = start.checked_add(usize::try_from(range[1]).ok()?)?;
+ let mut offset = 0;
+ let mut start_byte = None;
+ let mut end_byte = None;
+ for (byte, c) in value.char_indices().chain(std::iter::once((value.len(), '\0'))) {
+ if offset == start {
+ start_byte = Some(byte);
+ }
+ if offset == end {
+ end_byte = Some(byte);
+ }
+ if byte < value.len() {
+ offset += c.len_utf16();
+ }
+ }
+ let text: String = value[start_byte?..end_byte?].chars().take(limit).collect();
+ let shown_end = start + text.encode_utf16().count();
+ Some(VisibleText { text, start, end, shown_end, total: offset })
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct TextStyle {
+ pub bold: bool,
+ pub italic: bool,
+ pub underline: bool,
+ pub strike: bool,
+ pub link: Option<String>,
+ pub list_prefix: Option<String>,
+ pub list_level: usize,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct RenderedText {
+ pub markdown: String,
+ spans: Vec<(std::ops::Range<usize>, Option<std::ops::Range<usize>>)>,
+}
+
+impl RenderedText {
+ pub fn retained_bytes(&self) -> usize {
+ self.markdown.capacity() * 2
+ + self.spans.capacity() * std::mem::size_of::<(std::ops::Range<usize>, Option<std::ops::Range<usize>>)>()
+ + 2 * std::mem::size_of::<ObservedText>()
+ }
+
+ pub fn syntax(&mut self, value: &str) {
+ if !value.is_empty() {
+ let start = self.markdown.len();
+ self.markdown.push_str(value);
+ self.spans.push((start..self.markdown.len(), None));
+ }
+ }
+
+ fn native(&mut self, value: &str, escaped: bool, at: &mut usize) {
+ for c in value.chars() {
+ let start = self.markdown.len();
+ if escaped && c.is_ascii_punctuation() {
+ self.markdown.push('\\');
+ }
+ self.markdown.push(c);
+ let end = *at + c.len_utf16();
+ self.spans.push((start..self.markdown.len(), Some(*at..end)));
+ *at = end;
+ }
+ }
+
+ fn append(&mut self, other: Self) {
+ let offset = self.markdown.len();
+ self.markdown += &other.markdown;
+ self.spans.extend(other.spans.into_iter().map(|(range, native)| (range.start + offset..range.end + offset, native)));
+ }
+
+ fn wrap(self, open: &str, close: &str) -> Self {
+ let mut result = Self::default();
+ result.syntax(open);
+ result.append(self);
+ result.syntax(close);
+ result
+ }
+
+ pub fn offset(&mut self, offset: usize) {
+ for (_, native) in &mut self.spans {
+ if let Some(native) = native {
+ native.start += offset;
+ native.end += offset;
+ }
+ }
+ }
+
+ fn native_range(&self, start: usize, end: usize) -> Option<[isize; 2]> {
+ let spans: Vec<_> = self.spans.iter().filter(|(span, _)| span.start < end && span.end > start).collect();
+ if spans.first()?.0.start != start || spans.last()?.0.end != end {
+ return None;
+ }
+ let mut native = spans.iter().filter_map(|(_, native)| native.as_ref());
+ let first = native.next()?;
+ let mut end = first.end;
+ for next in native {
+ if next.start != end {
+ return None;
+ }
+ end = next.end;
+ }
+ Some([isize::try_from(first.start).ok()?, isize::try_from(end - first.start).ok()?])
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct ObservedText {
+ fingerprint: [u8; 32],
+ pub rendered: RenderedText,
+}
+
+impl ObservedText {
+ pub fn new(value: &str, rendered: RenderedText) -> Self {
+ Self { fingerprint: Sha256::digest(value.as_bytes()).into(), rendered }
+ }
+
+ fn matches(&self, value: &str) -> bool {
+ self.fingerprint == <[u8; 32]>::from(Sha256::digest(value.as_bytes()))
+ }
+}
+
+#[cfg(test)]
+pub fn attributed_markdown(runs: &[(String, TextStyle)], starts_line: bool) -> Option<String> {
+ attributed_text(runs, starts_line).map(|rendered| rendered.markdown)
+}
+
+pub fn attributed_text(runs: &[(String, TextStyle)], starts_line: bool) -> Option<RenderedText> {
+ if runs.iter().all(|(_, style)| *style == TextStyle::default()) {
+ return None;
+ }
+ let mut merged: Vec<(String, &TextStyle)> = vec![];
+ for (text, style) in runs {
+ if let Some((previous, _)) = merged.last_mut().filter(|(_, previous_style)| *previous_style == style) {
+ previous.push_str(text);
+ } else {
+ merged.push((text.clone(), style));
+ }
+ }
+ let mut out = RenderedText::default();
+ let mut native_at = 0;
+ let mut line_start = starts_line;
+ for (text, style) in merged {
+ for line in text.split_inclusive('\n') {
+ if line_start && !line.trim().is_empty() {
+ if let Some(prefix) = &style.list_prefix {
+ let marker = prefix.trim();
+ let ordered = marker
+ .strip_suffix('.')
+ .or_else(|| marker.strip_suffix(')'))
+ .filter(|digits| !digits.is_empty() && digits.chars().all(|c| c.is_ascii_digit()));
+ let marker = ordered.map_or_else(|| "-".to_string(), |digits| format!("{digits}."));
+ out.syntax(&format!("{}{marker} ", " ".repeat(style.list_level.min(20))));
+ }
+ }
+ let body = line.trim();
+ let leading = line.len() - line.trim_start().len();
+ out.native(&line[..leading], false, &mut native_at);
+ if !body.is_empty() {
+ let mut rendered = RenderedText::default();
+ rendered.native(body, true, &mut native_at);
+ let html_emphasis = out.markdown.ends_with('*')
+ || body.starts_with(|c: char| !c.is_alphanumeric())
+ || body.ends_with(|c: char| !c.is_alphanumeric());
+ let (bold_open, bold_close, italic_open, italic_close) =
+ if html_emphasis { ("<strong>", "</strong>", "<em>", "</em>") } else { ("**", "**", "*", "*") };
+ for (enabled, open, close) in [
+ (style.bold, bold_open, bold_close),
+ (style.italic, italic_open, italic_close),
+ (style.strike, "~~", "~~"),
+ (style.underline, "<u>", "</u>"),
+ ] {
+ if enabled {
+ rendered = rendered.wrap(open, close);
+ }
+ }
+ if let Some(url) = &style.link {
+ let url = url
+ .bytes()
+ .map(|b| if b <= 32 || b >= 127 || b"<>\\".contains(&b) { format!("%{b:02X}") } else { (b as char).to_string() })
+ .collect::<String>();
+ rendered = rendered.wrap("[", &format!("](<{url}>)"));
+ }
+ out.append(rendered);
+ out.native(&line[line.trim_end().len()..], false, &mut native_at);
+ }
+ line_start = line.ends_with('\n');
+ }
+ }
+ Some(out)
+}
+
+fn matches<'a>(value: &'a str, text: &'a str, prefix: &'a str, suffix: &'a str) -> impl Iterator<Item = usize> + 'a {
+ value
+ .char_indices()
+ .map(|(i, _)| i)
+ .filter(move |&i| value[i..].starts_with(text) && value[..i].ends_with(prefix) && value[i + text.len()..].starts_with(suffix))
+}
+
+#[cfg(test)]
+pub fn selection_range(value: &str, text: &str, prefix: &str, suffix: &str, selection: Selection) -> Result<[isize; 2], &'static str> {
+ observed_selection_range(value, None, text, prefix, suffix, selection)
+}
+
+pub fn observed_selection_range(
+ value: &str,
+ observed: Option<&ObservedText>,
+ text: &str,
+ prefix: &str,
+ suffix: &str,
+ selection: Selection,
+) -> Result<[isize; 2], &'static str> {
+ if text.is_empty() {
+ return Err("Selection text must not be empty");
+ }
+ if observed.is_some_and(|observed| !observed.matches(value)) {
+ return Err("The element's text changed since its observation; observe it again before selecting");
+ }
+ let mut ranges: Vec<_> = matches(value, text, prefix, suffix)
+ .take(2)
+ .map(|at| [value[..at].encode_utf16().count() as isize, text.encode_utf16().count() as isize])
+ .collect();
+ if let Some(observed) = observed {
+ ranges.extend(
+ matches(&observed.rendered.markdown, text, prefix, suffix).filter_map(|at| observed.rendered.native_range(at, at + text.len())),
+ );
+ }
+ ranges.sort_unstable();
+ ranges.dedup();
+ let [start, len] = match ranges.as_slice() {
+ [range] => *range,
+ [] => return Err("The requested text and context were not found in the element's value or observed Markdown"),
+ _ => return Err("The requested text matches more than once; provide prefix or suffix to identify one occurrence"),
+ };
+ Ok(match selection {
+ Selection::Text => [start, len],
+ Selection::CursorBefore => [start, 0],
+ Selection::CursorAfter => [start + len, 0],
+ })
+}
+
+fn strip_html(html: &str) -> String {
+ html2text::config::plain_no_decorate()
+ .string_from_read(html.as_bytes(), usize::MAX / 2)
+ .map(|t| t.trim().to_string())
+ .unwrap_or_else(|_| html.to_string())
+}
+
+#[cfg(test)]
+mod tests {
+ fn observed(runs: Vec<(String, TextStyle)>) -> (String, ObservedText) {
+ let value: String = runs.iter().map(|(text, _)| text.as_str()).collect();
+ let observed = ObservedText::new(&value, attributed_text(&runs, true).unwrap());
+ (value, observed)
+ }
+
+ #[test]
+ fn observed_selection_freshness_retains_no_full_value() {
+ let rendered = attributed_text(&[("hello".into(), TextStyle { bold: true, ..Default::default() })], true).unwrap();
+ let small_cost = ObservedText::new("hello", rendered.clone()).rendered.retained_bytes();
+ let mut large = "hello".to_string() + &"x".repeat(2 * 1024 * 1024);
+ let observed = ObservedText::new(&large, rendered);
+ assert_eq!(observed.rendered.retained_bytes(), small_cost);
+ assert_eq!(observed_selection_range(&large, Some(&observed), "**hello**", "", "", Selection::Text).unwrap(), [0, 5]);
+ large.pop();
+ large.push('y');
+ assert!(observed_selection_range(&large, Some(&observed), "**hello**", "", "", Selection::Text).is_err());
+ }
+
+ #[test]
+ fn observed_selection_maps_escaped_unicode_and_link_labels() {
+ let (value, observed) = observed(vec![
+ ("😀 ".into(), TextStyle::default()),
+ ("[label]".into(), TextStyle { link: Some("https://example.test/target".into()), ..Default::default() }),
+ ]);
+ let choose = |text, selection| observed_selection_range(&value, Some(&observed), text, "", "", selection);
+ assert_eq!(choose("[\\[label\\]](<https://example.test/target>)", Selection::Text).unwrap(), [3, 7]);
+ assert_eq!(choose("\\[label\\]", Selection::CursorAfter).unwrap(), [10, 0]);
+ assert_eq!(choose("label", Selection::Text).unwrap(), [4, 5]);
+ assert!(choose("https://example.test/target", Selection::Text).is_err());
+ assert!(choose("[label\\]", Selection::Text).is_err());
+ }
+
+ #[test]
+ fn observed_selection_disambiguates_formatting_without_guessing_literal_markup() {
+ let bold = TextStyle { bold: true, ..Default::default() };
+ let (value, observed) = observed(vec![("foo ".into(), TextStyle::default()), ("foo".into(), bold.clone())]);
+ assert!(observed_selection_range(&value, Some(&observed), "foo", "", "", Selection::Text).is_err());
+ assert_eq!(observed_selection_range(&value, Some(&observed), "**foo**", "", "", Selection::Text).unwrap(), [4, 3]);
+ assert_eq!(observed_selection_range(&value, Some(&observed), "foo", "**", "**", Selection::Text).unwrap(), [4, 3]);
+ let (literal_value, literal) = self::observed(vec![("**foo** ".into(), TextStyle::default()), ("foo".into(), bold)]);
+ assert!(observed_selection_range(&literal_value, Some(&literal), "**foo**", "", "", Selection::Text).is_err());
+ assert_eq!(observed_selection_range(&literal_value, Some(&literal), "**foo**", " ", "", Selection::Text).unwrap(), [8, 3]);
+ }
+
+ #[test]
+ fn observed_selection_uses_visible_offsets_and_rejects_stale_text_and_notices() {
+ let (_, mut observed) = observed(vec![("😀bold".into(), TextStyle { bold: true, ..Default::default() })]);
+ observed.rendered.offset(7);
+ observed.rendered.syntax("\n… text truncated");
+ let value = "prefix 😀bold tail";
+ let observed = ObservedText::new(value, observed.rendered);
+ assert_eq!(observed_selection_range(value, Some(&observed), "😀bold", "", "", Selection::Text).unwrap(), [7, 6]);
+ assert!(observed_selection_range(value, Some(&observed), "text truncated", "", "", Selection::Text).is_err());
+ assert!(observed_selection_range("changed 😀bold tail", Some(&observed), "😀bold", "", "", Selection::Text).is_err());
+ }
+
+ #[test]
+ fn attributed_observation_preserves_paragraphs_lists_and_escaped_links() {
+ let runs = vec![
+ ("first\n\n".into(), TextStyle { bold: true, ..Default::default() }),
+ ("item".into(), TextStyle { italic: true, list_prefix: Some("•".into()), list_level: 1, ..Default::default() }),
+ (" more\n".into(), TextStyle { list_prefix: Some("•".into()), list_level: 1, ..Default::default() }),
+ ("[link]".into(), TextStyle { link: Some("https://example.test/é>a b".into()), ..Default::default() }),
+ ];
+ assert_eq!(
+ attributed_markdown(&runs, true).unwrap(),
+ "**first**\n\n - *item* more\n[\\[link\\]](<https://example.test/%C3%A9%3Ea%20b>)"
+ );
+ assert_eq!(attributed_markdown(&runs[1..2], false).unwrap(), "*item*");
+ assert!(attributed_markdown(&[("plain".into(), TextStyle::default())], true).is_none());
+ }
+
+ #[test]
+ fn attributed_observation_emphasizes_inside_words() {
+ let runs = vec![
+ ("a".into(), TextStyle::default()),
+ ("b".into(), TextStyle { italic: true, ..Default::default() }),
+ ("c".into(), TextStyle::default()),
+ ];
+ let markdown = attributed_markdown(&runs, true).unwrap();
+ let mut html = String::new();
+ pulldown_cmark::html::push_html(&mut html, pulldown_cmark::Parser::new(&markdown));
+ assert_eq!(html, "<p>a<em>b</em>c</p>\n");
+ for (middle, suffix, expected) in [
+ ("(", TextStyle::default(), "<p>a<em>(</em>c</p>\n"),
+ ("b", TextStyle { bold: true, ..Default::default() }, "<p>a<em>b</em><strong>c</strong></p>\n"),
+ ] {
+ let runs = vec![
+ ("a".into(), TextStyle::default()),
+ (middle.into(), TextStyle { italic: true, ..Default::default() }),
+ ("c".into(), suffix),
+ ];
+ let markdown = attributed_markdown(&runs, true).unwrap();
+ let mut html = String::new();
+ pulldown_cmark::html::push_html(&mut html, pulldown_cmark::Parser::new(&markdown));
+ assert_eq!(html, expected);
+ }
+ }
+
+ #[test]
+ fn attributed_observation_coalesces_equivalent_runs() {
+ let style = TextStyle { bold: true, strike: true, underline: true, ..Default::default() };
+ let runs = vec![("bo".into(), style.clone()), ("ld".into(), style)];
+ assert_eq!(attributed_markdown(&runs, true).unwrap(), "<u>~~**bold**~~</u>");
+ }
+
+ use super::*;
+
+ fn us(code: u16, shift: bool, option: bool) -> Option<char> {
+ let base = match code {
+ 0 => 'a',
+ 9 => 'v',
+ 18 => '1',
+ 29 => '0',
+ 24 => '=',
+ 49 => ' ',
+ 83 => '1',
+ 14 if option => return None,
+ 14 => 'e',
+ _ => return None,
+ };
+ Some(match (base, shift, option) {
+ ('1', true, false) => '!',
+ ('=', true, false) => '+',
+ ('a', false, true) => 'å',
+ ('a', true, true) => 'Å',
+ (c, true, false) => c.to_ascii_uppercase(),
+ (c, _, _) => c,
+ })
+ }
+
+ #[test]
+ fn layout_prefers_fewest_modifiers_and_main_keys() {
+ let map = layout(us);
+ assert_eq!(map[&'a'], Stroke { code: 0, shift: false, option: false });
+ assert_eq!(map[&'A'], Stroke { code: 0, shift: true, option: false });
+ assert_eq!(map[&'!'], Stroke { code: 18, shift: true, option: false });
+ assert_eq!(map[&'å'], Stroke { code: 0, shift: false, option: true });
+ assert_eq!(map[&'1'].code, 18, "the main-row 1, not the keypad 1");
+ assert_eq!(map[&'+'], Stroke { code: 24, shift: true, option: false });
+ assert!(!map.values().any(|s| s.code == 14 && s.option), "dead keys are never used");
+ }
+
+ #[test]
+ fn plan_uses_real_keys_and_presses_return_and_tab() {
+ let map = layout(us);
+ let keys = plan("Ab\tA\r\n!\n😀", &map);
+ assert_eq!(
+ keys,
+ vec![
+ Key::Mapped('A', Stroke { code: 0, shift: true, option: false }),
+ Key::Unicode('b'),
+ Key::Tab,
+ Key::Mapped('A', Stroke { code: 0, shift: true, option: false }),
+ Key::Return,
+ Key::Mapped('!', Stroke { code: 18, shift: true, option: false }),
+ Key::Return,
+ Key::Unicode('😀'),
+ ]
+ );
+ assert_eq!(plan("a\r\rb", &map).iter().filter(|k| **k == Key::Return).count(), 2);
+ assert_eq!(plan("x", &HashMap::new()), vec![Key::Unicode('x')], "an unreadable layout falls back to text");
+ }
+
+ #[test]
+ fn payload_formats() {
+ assert_eq!(payload("a\nb", PasteFormat::Text), ("a\nb".into(), None));
+ let (plain, html) = payload("**hi** there", PasteFormat::Markdown);
+ assert_eq!(plain, "**hi** there");
+ assert!(html.unwrap().contains("<strong>hi</strong> there"));
+ let (plain, html) = payload("<p>One &amp; <b>two</b></p><p>three<br>four</p>", PasteFormat::Html);
+ assert_eq!(plain, "One & two\n\nthree\nfour");
+ assert!(html.unwrap().ends_with("<p>One &amp; <b>two</b></p><p>three<br>four</p>"));
+ }
+
+ #[test]
+ fn paste_format_defaults_to_text() {
+ let a: crate::protocol::Action = serde_json::from_str(r#"{"type":"paste","text":"x"}"#).unwrap();
+ assert!(matches!(a, crate::protocol::Action::Paste { format: PasteFormat::Text, .. }));
+ }
+ #[test]
+ fn insertion_must_match_the_pasted_text_at_the_selection() {
+ assert!(inserted("ab", Some((1, 0)), "aXYb", "XY"));
+ assert!(inserted("a old b", Some((2, 3)), "a new b", "new"), "replaces the selection");
+ assert!(inserted("ab", None, "aXYb", "XY"), "unknown selection: anywhere");
+ assert!(inserted("", Some((0, 0)), "line1\rline2", "line1\nline2"), "line endings normalised");
+ assert!(inserted("a", Some((1, 0)), "a ", " "), "a whitespace paste");
+ assert!(!inserted("x", Some((1, 0)), "x one\n two", "one two"), "whitespace must match");
+ assert!(!inserted("ab", Some((1, 0)), "ab", "XY"), "nothing changed");
+ assert!(!inserted("ab", Some((1, 0)), "aZb", "XY"), "something else changed");
+ assert!(!inserted("ab", Some((0, 0)), "aXYb", "XY"), "inserted at the wrong place");
+ assert!(!inserted("aXb", Some((1, 1)), "aXb", "X"), "replacing a selection with itself is no evidence");
+ assert!(!inserted("ab", None, "aXYbZ", "XY"), "unknown selection still needs nothing else to change");
+ assert!(!inserted("ab", Some((5, 0)), "aXYb", "XY"), "stale selection");
+ assert!(!inserted("ab", Some((1, 0)), "ab", ""), "nothing expected");
+ assert!(inserted("é", Some((1, 0)), "é😀", "😀"), "UTF-16 offsets");
+ assert!(inserted("aba", None, "ababa", "ab"));
+ assert!(inserted("éé", None, "é😀é", "😀"));
+ assert!(inserted("é", None, "éXĩ", "Xĩ"));
+ assert!(!inserted("éé", None, "éêé", "😀"));
+ assert_eq!(expected("**hi** there", PasteFormat::Markdown), vec!["**hi** there".to_string(), "hi there".to_string()]);
+ assert_eq!(expected("plain", PasteFormat::Text), vec!["plain".to_string()]);
+ }
+
+ #[test]
+ fn long_paste_confirmation_without_selection_finishes_promptly() {
+ let before = "a".repeat(2_097_152);
+ let pasted = "b".repeat(262_144);
+ let after = format!("{before}{pasted}");
+ let started = std::time::Instant::now();
+ assert!(inserted(&before, None, &after, &pasted));
+ assert!(started.elapsed() < std::time::Duration::from_secs(5));
+ }
+
+ #[test]
+ fn selection_requires_unique_context_including_overlapping_matches() {
+ assert!(selection_range("aaa", "aa", "", "", Selection::Text).is_err());
+ assert_eq!(selection_range("aaa", "aa", "a", "", Selection::Text), Ok([1, 2]));
+ assert_eq!(selection_range("one cat; two cat!", "cat", "two ", "!", Selection::Text), Ok([13, 3]));
+ assert!(selection_range("one cat", "cat", "two ", "", Selection::Text).is_err());
+ }
+
+ #[test]
+ fn selection_and_carets_use_utf16_offsets() {
+ let value = "é😀a界!";
+ assert_eq!(selection_range(value, "a界", "😀", "!", Selection::Text), Ok([3, 2]));
+ assert_eq!(selection_range(value, "a界", "", "", Selection::CursorBefore), Ok([3, 0]));
+ assert_eq!(selection_range(value, "a界", "", "", Selection::CursorAfter), Ok([5, 0]));
+ assert_eq!(selection_range(value, "😀", "", "", Selection::Text), Ok([1, 2]));
+ assert!(selection_range(value, "", "", "", Selection::CursorAfter).is_err());
+ assert!(selection_range(value, "", "!", "", Selection::CursorAfter).is_err());
+ }
+
+ #[test]
+ fn set_value_conversions() {
+ assert_eq!(flag(" Yes "), Some(true));
+ assert_eq!(flag("0"), Some(false));
+ assert_eq!(flag("maybe"), None);
+ assert_eq!(number("2.5"), Some(2.5));
+ assert_eq!(number("true"), Some(1.0));
+ assert_eq!(number("NaN"), None);
+ assert_eq!(number("ten"), None);
+ }
+}
diff --git a/desktop/desk/src/tree.rs b/desktop/desk/src/tree.rs
new file mode 100644
index 000000000..f88bf65fa
--- /dev/null
+++ b/desktop/desk/src/tree.rs
@@ -0,0 +1,1145 @@
+use crate::protocol::Rect;
+use std::collections::{HashMap, HashSet};
+
+const EDGE: f64 = 1.0;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum Key {
+ Element(u32),
+ Note(u32, u32),
+ Section(u32),
+ Info(u32),
+}
+
+pub const ROOT: u32 = u32::MAX;
+
+pub fn intersect(a: Rect, b: Rect) -> Option<Rect> {
+ let (x, y) = (a.x.max(b.x), a.y.max(b.y));
+ let (right, bottom) = ((a.x + a.w).min(b.x + b.w), (a.y + a.h).min(b.y + b.h));
+ (right > x && bottom > y).then_some(Rect { x, y, w: right - x, h: bottom - y })
+}
+
+pub fn visible(f: Rect, clip: Rect) -> Option<Rect> {
+ if f.w > 0.0 && f.h > 0.0 {
+ return intersect(f, clip);
+ }
+ let inside = f.x >= clip.x && f.x <= clip.x + clip.w && f.y > clip.y + EDGE && f.y < clip.y + clip.h - EDGE;
+ inside.then_some(f)
+}
+
+pub struct Rendered {
+ pub tree: String,
+ pub diffed: bool,
+ pub removed: Vec<u32>,
+}
+
+pub fn render(lines: &[(Key, String)], base: Option<&HashMap<Key, String>>, moved: &HashSet<u32>) -> Rendered {
+ let full = || Rendered { tree: lines.iter().map(|(_, l)| format!("{l}\n")).collect(), diffed: false, removed: vec![] };
+ let Some(base) = base else { return full() };
+ let mut tree = String::new();
+ let mut changed = 0;
+ for (key, line) in lines {
+ let rebound = matches!(key, Key::Element(id) if moved.contains(id));
+ let mark = match base.get(key) {
+ Some(old) if old == line && !rebound => continue,
+ Some(_) => "~",
+ None => "+",
+ };
+ changed += 1;
+ tree += &format!("{mark} {line}\n");
+ }
+ if changed * 5 > lines.len() * 3 {
+ return full();
+ }
+ let present: HashSet<&Key> = lines.iter().map(|(k, _)| k).collect();
+ if base.keys().any(|key| !matches!(key, Key::Element(_)) && !present.contains(key)) {
+ return full();
+ }
+ let mut removed: Vec<u32> = base
+ .keys()
+ .filter(|k| !present.contains(k))
+ .filter_map(|k| match k {
+ Key::Element(id) => Some(*id),
+ _ => None,
+ })
+ .collect();
+ removed.sort_unstable();
+ Rendered { tree, diffed: true, removed }
+}
+
+const TEXT_ROLES: &[&str] = &["AXStaticText", "AXTextArea", "AXTextField", "AXHeading"];
+const TOGGLES: &[&str] = &["AXCheckBox", "AXRadioButton", "AXSwitch", "AXToggle", "AXMenuItem"];
+const QUIET_SUBROLES: &[&str] = &["AXUnknown", "AXStandardWindow", "AXTextAttachment"];
+pub const MAX_TEXT: usize = 1000;
+const FOLDS: &[&str] = &["AXHeading", "AXLink"];
+
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct Props {
+ pub role: String,
+ pub subrole: Option<String>,
+ pub title: Option<String>,
+ pub description: Option<String>,
+ pub label: Option<String>,
+ pub value: Option<String>,
+ pub rich_text: Option<String>,
+ pub visible_text: Option<crate::text::VisibleText>,
+ pub placeholder: Option<String>,
+ pub url: Option<String>,
+ pub identifier: Option<String>,
+ pub expanded: Option<bool>,
+ pub focused: bool,
+ pub disabled: bool,
+ pub selected: bool,
+ pub settable: bool,
+ pub actions: Vec<String>,
+ pub list_item: bool,
+}
+
+pub fn clip(s: &str, n: usize) -> String {
+ let s = s.replace('\n', "⏎");
+ if s.chars().count() > n {
+ format!("{}…", s.chars().take(n).collect::<String>())
+ } else {
+ s
+ }
+}
+
+fn short(role: &str) -> &str {
+ role.strip_prefix("AX").unwrap_or(role)
+}
+
+impl Props {
+ pub fn name(&self) -> Option<&str> {
+ self.title.as_deref().or(self.label.as_deref())
+ }
+
+ fn toggle(&self) -> Option<&'static str> {
+ if !TOGGLES.contains(&self.role.as_str()) && self.subrole.as_deref() != Some("AXSwitch") {
+ return None;
+ }
+ match self.value.as_deref() {
+ Some("1") => Some("checked"),
+ Some("2") => Some("mixed"),
+ Some("0") if self.role != "AXMenuItem" => Some("unchecked"),
+ _ => None,
+ }
+ }
+
+ fn heading_level(&self) -> Option<usize> {
+ let level = self.value.as_deref()?.parse::<usize>().ok()?;
+ (self.role == "AXHeading" && (1..=6).contains(&level)).then_some(level)
+ }
+
+ pub fn folds_text(&self) -> bool {
+ (self.list_item || FOLDS.contains(&self.role.as_str())) && self.title.is_none() && self.label.is_none()
+ }
+
+ pub fn content_list(&self) -> bool {
+ self.role == "AXList" && self.subrole.as_deref() == Some("AXContentList")
+ }
+
+ pub fn plain_text(&self) -> bool {
+ self.role == "AXStaticText"
+ && self.rich_text.is_none()
+ && self.actions.is_empty()
+ && !self.focused
+ && !self.selected
+ && self.text().is_some()
+ }
+
+ pub fn text(&self) -> Option<&str> {
+ self.value.as_deref().or(self.title.as_deref()).or(self.description.as_deref())
+ }
+
+ pub fn describe(&self) -> (String, bool) {
+ let toggle = self.toggle();
+ let name = self.name();
+ let value = self
+ .visible_text
+ .as_ref()
+ .map(|v| v.text.as_str())
+ .or(self.value.as_deref())
+ .filter(|v| Some(*v) != name && toggle.is_none() && self.heading_level().is_none());
+ let description = self.description.as_deref().filter(|d| Some(*d) != name);
+ let placeholder = self.placeholder.as_deref().filter(|_| value.is_none());
+ let keep = self.list_item
+ || name.is_some()
+ || description.is_some()
+ || value.is_some()
+ || placeholder.is_some()
+ || self.url.is_some()
+ || toggle.is_some()
+ || self.expanded.is_some()
+ || self.focused
+ || self.selected
+ || self.settable
+ || !self.actions.is_empty();
+ let level = self.heading_level().filter(|_| name.is_some());
+ let bullet = self.list_item && self.role == "AXGroup";
+ let markdown = level.is_some() || bullet;
+ let mut s = match (level, bullet) {
+ (Some(l), _) => format!("{} {}", "#".repeat(l), clip(name.unwrap_or_default(), MAX_TEXT)),
+ (None, true) => name.map_or("-".to_string(), |n| format!("- {}", clip(n, MAX_TEXT))),
+ _ => short(&self.role).to_string(),
+ };
+ if let Some(sub) = self.subrole.as_deref().filter(|s| !QUIET_SUBROLES.contains(s) && !markdown) {
+ s += &format!(":{}", short(sub));
+ }
+ if let Some(n) = name.filter(|_| !markdown) {
+ s += &format!(" \"{}\"", clip(n, 80));
+ }
+ if let Some(d) = description {
+ s += &format!(" ({})", clip(d, 80));
+ }
+ if let Some(v) = value {
+ let n = if TEXT_ROLES.contains(&self.role.as_str()) { MAX_TEXT } else { 120 };
+ if self.rich_text.is_none() {
+ s += &format!(" = \"{}\"", clip(v, n));
+ }
+ if let Some(v) = &self.visible_text {
+ s += &format!(" [visible UTF-16 range {}..{} of {}", v.start, v.end, v.total);
+ if v.shown_end < v.end {
+ s += &format!("; shown {}..{}, truncated", v.start, v.shown_end);
+ }
+ s.push(']');
+ }
+ }
+ if let Some(p) = placeholder {
+ s += &format!(" placeholder \"{}\"", clip(p, 80));
+ }
+ if let Some(u) = &self.url {
+ s += &format!(" url \"{}\"", clip(u, 160));
+ }
+ let mut tags = vec![];
+ if self.focused {
+ tags.push("focused");
+ }
+ if self.disabled {
+ tags.push("disabled");
+ }
+ if self.selected {
+ tags.push("selected");
+ }
+ tags.extend(toggle);
+ match self.expanded {
+ Some(true) => tags.push("expanded"),
+ Some(false) => tags.push("collapsed"),
+ None => {}
+ }
+ if self.settable {
+ tags.push("settable");
+ }
+ if !tags.is_empty() {
+ s += &format!(" [{}]", tags.join(" "));
+ }
+ let secondary: Vec<&str> = self.actions.iter().filter(|a| a.as_str() != "AXPress").map(|a| short(a)).collect();
+ if !secondary.is_empty() {
+ s += &format!(" <{}>", secondary.join(","));
+ }
+ if let Some(markdown) = &self.rich_text {
+ s += &format!(" markdown:\n{markdown}");
+ }
+ (s, keep)
+ }
+}
+
+pub fn joined_text(parts: &[(String, Option<Rect>)]) -> String {
+ let mut out = String::new();
+ let mut last: Option<Rect> = None;
+ for (text, f) in parts {
+ let wrapped = matches!((last, f), (Some(a), Some(b)) if b.y >= a.y + a.h - EDGE);
+ if wrapped && !out.is_empty() && !out.ends_with(char::is_whitespace) && !text.starts_with(char::is_whitespace) {
+ out.push('\n');
+ }
+ out += text;
+ last = f.or(last);
+ }
+ out.trim().to_string()
+}
+
+pub fn merged_text(parts: &[(String, Option<Rect>)]) -> String {
+ format!("StaticText = \"{}\"", clip(&joined_text(parts), MAX_TEXT))
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Item {
+ pub key: Key,
+ pub text: String,
+ pub kids: Vec<Item>,
+}
+
+impl Item {
+ pub fn leaf(key: Key, text: String) -> Item {
+ Item { key, text, kids: vec![] }
+ }
+}
+
+const MAX_INDENT: usize = 50;
+const CUT_NOTE: usize = 80;
+
+fn indent(depth: usize) -> String {
+ " ".repeat(depth.min(MAX_INDENT))
+}
+
+impl Item {
+ fn line(&self, depth: usize) -> String {
+ let text = self.text.replace('\n', &format!("\n{}", indent(depth + 1)));
+ match self.key {
+ Key::Element(id) => format!("{}[{id}] {}", indent(depth), text),
+ _ => format!("{}{}", indent(depth), text),
+ }
+ }
+
+ pub fn renumber(&mut self, moved: &HashMap<u32, u32>) {
+ if let Key::Element(id) = self.key {
+ self.key = Key::Element(*moved.get(&id).unwrap_or(&id));
+ }
+ if let Key::Note(a, n) = self.key {
+ self.key = Key::Note(*moved.get(&a).unwrap_or(&a), n);
+ }
+ self.kids.iter_mut().for_each(|k| k.renumber(moved));
+ }
+}
+
+fn own(it: &Item, depth: usize) -> usize {
+ it.line(depth).len() + 1
+}
+
+fn size(it: &Item, depth: usize) -> usize {
+ own(it, depth) + it.kids.iter().map(|k| size(k, depth + 1)).sum::<usize>()
+}
+
+fn anchor_of(key: Key, parent: u32) -> u32 {
+ match key {
+ Key::Element(id) => id,
+ _ => parent,
+ }
+}
+
+fn emit(it: &Item, depth: usize, out: &mut Vec<(Key, String)>) {
+ out.push((it.key, it.line(depth)));
+ for k in &it.kids {
+ emit(k, depth + 1, out);
+ }
+}
+
+fn cut_note(anchor: u32, depth: usize, n: usize, out: &mut Vec<(Key, String)>) {
+ out.push((Key::Note(anchor, u32::MAX), format!("{}… {n} more not shown; the observation size limit was reached", indent(depth))));
+}
+
+fn fit(items: &[Item], depth: usize, anchor: u32, budget: usize, out: &mut Vec<(Key, String)>) -> bool {
+ let sizes: Vec<usize> = items.iter().map(|i| size(i, depth)).collect();
+ if sizes.iter().sum::<usize>() <= budget {
+ items.iter().for_each(|i| emit(i, depth, out));
+ return false;
+ }
+ let floor: Vec<usize> =
+ items.iter().map(|i| own(i, depth) + if i.kids.is_empty() { 0 } else { indent(depth + 1).len() + CUT_NOTE }).collect();
+ if floor.iter().sum::<usize>() > budget {
+ let room = budget.saturating_sub(indent(depth).len() + CUT_NOTE);
+ let mut used = 0;
+ let mut shown = 0;
+ for (i, it) in items.iter().enumerate() {
+ if used + floor[i] > room {
+ break;
+ }
+ used += floor[i];
+ shown += 1;
+ out.push((it.key, it.line(depth)));
+ if !it.kids.is_empty() {
+ cut_note(anchor_of(it.key, anchor), depth + 1, it.kids.len(), out);
+ }
+ }
+ cut_note(anchor, depth, items.len() - shown, out);
+ return true;
+ }
+ let allowance = |cap: usize| -> Vec<usize> { sizes.iter().zip(&floor).map(|(&s, &f)| s.min(cap).max(f)).collect() };
+ let (mut lo, mut hi) = (0, *sizes.iter().max().unwrap_or(&0));
+ while lo < hi {
+ let mid = (lo + hi).div_ceil(2);
+ if allowance(mid).iter().sum::<usize>() <= budget {
+ lo = mid;
+ } else {
+ hi = mid - 1;
+ }
+ }
+ for (it, give) in items.iter().zip(allowance(lo)) {
+ if size(it, depth) <= give {
+ emit(it, depth, out);
+ } else {
+ out.push((it.key, it.line(depth)));
+ fit(&it.kids, depth + 1, anchor_of(it.key, anchor), give - own(it, depth), out);
+ }
+ }
+ true
+}
+
+pub fn layout(items: &[Item], budget: usize) -> (Vec<(Key, String)>, bool) {
+ let mut out = vec![];
+ let cut = fit(items, 0, ROOT, budget, &mut out);
+ (out, cut)
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Locator {
+ hash: u64,
+ pub rebindable: bool,
+}
+
+const COLLECTIONS: &[&str] = &["AXRow", "AXOutlineRow", "AXCell", "AXList", "AXTable", "AXOutline", "AXBrowser", "AXGrid"];
+
+impl Locator {
+ pub const ROOT: Locator = Locator { hash: 0xcbf2_9ce4_8422_2325, rebindable: true };
+
+ pub fn child(self, p: &Props) -> Locator {
+ let mut h = self.hash;
+ let mut eat = |s: &str| {
+ for b in s.bytes().chain([0xff]) {
+ h ^= b as u64;
+ h = h.wrapping_mul(0x0100_0000_01b3);
+ }
+ };
+ eat(&p.role);
+ eat(p.subrole.as_deref().unwrap_or(""));
+ eat(p.identifier.as_deref().unwrap_or(""));
+ eat(p.name().unwrap_or(""));
+ if p.name().is_none() && p.identifier.is_none() {
+ eat(p.description.as_deref().unwrap_or(""));
+ }
+ Locator { hash: h, rebindable: self.rebindable && !COLLECTIONS.contains(&p.role.as_str()) }
+ }
+}
+
+pub fn path_locator<T: PartialEq>(
+ element: &T,
+ roots: &[(T, Locator)],
+ max_depth: usize,
+ expected_path: &[T],
+ mut read: impl FnMut(&T) -> Option<(Props, Vec<T>)>,
+) -> Option<Locator> {
+ if expected_path.last() != Some(element) || expected_path.len() > max_depth + 2 {
+ return None;
+ }
+ let (root_at, mut locator) = expected_path
+ .iter()
+ .enumerate()
+ .find_map(|(at, element)| roots.iter().find(|(root, _)| root == element).map(|(_, locator)| (at, *locator)))?;
+ if root_at > 1 {
+ return None;
+ }
+ for (at, current) in expected_path.iter().enumerate() {
+ if expected_path[..at].contains(current) {
+ return None;
+ }
+ let (props, children) = read(current)?;
+ if expected_path.get(at + 1).is_some_and(|next| !children.contains(next)) {
+ return None;
+ }
+ if at >= root_at {
+ locator = locator.child(&props);
+ }
+ }
+ Some(locator)
+}
+
+const PARK_OBSERVATIONS: u32 = 30;
+const MAX_PARKED: usize = 5000;
+
+struct Entry<T> {
+ id: u32,
+ el: T,
+ hash: usize,
+ locator: Locator,
+ idle: u32,
+ now: bool,
+}
+
+pub struct Ids<T> {
+ known: Vec<Entry<T>>,
+ by_hash: HashMap<usize, Vec<usize>>,
+ by_locator: HashMap<Locator, Vec<usize>>,
+ taken: HashSet<u32>,
+ fresh: Vec<(u32, Locator)>,
+ pub next: u32,
+}
+
+impl<T: PartialEq + Clone> Ids<T> {
+ pub fn new() -> Self {
+ Ids { known: vec![], by_hash: HashMap::new(), by_locator: HashMap::new(), taken: HashSet::new(), fresh: vec![], next: 0 }
+ }
+
+ fn add(&mut self, e: Entry<T>) {
+ let i = self.known.len();
+ self.by_hash.entry(e.hash).or_default().push(i);
+ self.by_locator.entry(e.locator).or_default().push(i);
+ self.known.push(e);
+ }
+
+ pub fn assign(&mut self, el: &T, hash: usize, locator: Locator) -> u32 {
+ let same = self.by_hash.get(&hash).and_then(|c| {
+ c.iter().copied().find(|&i| {
+ !self.known[i].now && !self.taken.contains(&self.known[i].id) && self.known[i].el == *el && self.known[i].locator == locator
+ })
+ });
+ let id = match same {
+ Some(i) => self.known[i].id,
+ None => {
+ self.next += 1;
+ self.fresh.push((self.next - 1, locator));
+ self.next - 1
+ }
+ };
+ self.taken.insert(id);
+ self.add(Entry { id, el: el.clone(), hash, locator, idle: 0, now: true });
+ id
+ }
+
+ pub fn rebind(&mut self, dead: impl Fn(&T) -> bool) -> HashMap<u32, u32> {
+ let mut count: HashMap<Locator, usize> = HashMap::new();
+ for entry in self.known.iter().filter(|entry| entry.now) {
+ *count.entry(entry.locator).or_default() += 1;
+ }
+ let mut moved = HashMap::new();
+ for (new, locator) in std::mem::take(&mut self.fresh) {
+ if count[&locator] != 1 || !locator.rebindable {
+ continue;
+ }
+ let old: HashSet<u32> = self.by_locator[&locator]
+ .iter()
+ .map(|&i| &self.known[i])
+ .filter(|e| !e.now && !self.taken.contains(&e.id))
+ .map(|e| e.id)
+ .collect();
+ let [id] = old.into_iter().collect::<Vec<_>>()[..] else { continue };
+ let Some(replacement) = self.known.iter().find(|entry| entry.now && entry.id == new) else { continue };
+ if self.replacement_owned_by_other(id, &replacement.el) {
+ continue;
+ }
+ let gone = self.by_locator[&locator].iter().filter(|&&i| self.known[i].id == id).all(|&i| dead(&self.known[i].el));
+ if !gone {
+ continue;
+ }
+ for &i in &self.by_locator[&locator] {
+ if self.known[i].id == new && self.known[i].now {
+ self.known[i].id = id;
+ }
+ }
+ self.taken.remove(&new);
+ self.taken.insert(id);
+ moved.insert(new, id);
+ }
+ moved
+ }
+
+ pub fn seen(&self, el: &T, hash: usize) -> bool {
+ self.by_hash.get(&hash).is_some_and(|c| c.iter().any(|&i| self.known[i].now && self.known[i].el == *el))
+ }
+
+ pub fn carry(self) -> Ids<T> {
+ let mut next = Ids::new();
+ next.next = self.next;
+ let mut seen = HashSet::new();
+ let (fresh, old): (Vec<_>, Vec<_>) = self.known.into_iter().partition(|e| e.now);
+ for mut e in fresh {
+ e.now = false;
+ if seen.insert(e.id) {
+ next.add(e);
+ }
+ }
+ for mut e in old {
+ if next.known.len() >= MAX_PARKED {
+ break;
+ }
+ if self.taken.contains(&e.id) || !seen.insert(e.id) || e.idle >= PARK_OBSERVATIONS {
+ continue;
+ }
+ e.idle += 1;
+ next.add(e);
+ }
+ next
+ }
+
+ fn replacement_owned_by_other(&self, id: u32, element: &T) -> bool {
+ self.known.iter().any(|entry| !entry.now && entry.id != id && entry.el == *element)
+ }
+
+ pub fn relocate(&self, id: u32, find: impl FnOnce(Locator) -> Option<T>) -> Option<T> {
+ let locator = self.locator(id)?;
+ if !locator.rebindable || self.by_locator.get(&locator)?.iter().any(|&at| self.known[at].id != id) {
+ return None;
+ }
+ let replacement = find(locator)?;
+ (!self.replacement_owned_by_other(id, &replacement)).then_some(replacement)
+ }
+
+ pub fn locator(&self, id: u32) -> Option<Locator> {
+ self.known.iter().find(|e| e.id == id).map(|e| e.locator)
+ }
+}
+
+impl<T: PartialEq + Clone> Default for Ids<T> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+pub fn sparse(region: Rect, window: Rect, lines_inside: usize) -> bool {
+ let area = |r: Rect| r.w.max(0.0) * r.h.max(0.0);
+ lines_inside == 0 && area(window) > 0.0 && area(region) >= 0.4 * area(window)
+}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn attributed_observation_keeps_one_control_and_native_text() {
+ let p = super::Props {
+ role: "AXStaticText".into(),
+ value: Some("first\nsecond".into()),
+ rich_text: Some("**first**\nsecond".into()),
+ ..Default::default()
+ };
+ assert!(!p.plain_text());
+ assert_eq!(p.text(), Some("first\nsecond"));
+ assert_eq!(p.describe(), ("StaticText markdown:\n**first**\nsecond".into(), true));
+ let item = super::Item::leaf(super::Key::Element(3), p.describe().0);
+ assert_eq!(item.line(1), " [3] StaticText markdown:\n **first**\n second");
+ }
+
+ #[test]
+ fn visible_text_observation_preserves_native_offsets_and_value() {
+ let value = " 😀old\nvisible text\ntail";
+ let p = super::Props {
+ role: "AXTextArea".into(),
+ value: Some(value.into()),
+ visible_text: crate::text::visible_text(value, [8, 12], super::MAX_TEXT),
+ ..Default::default()
+ };
+ assert_eq!(p.describe().0, "TextArea = \"visible text\" [visible UTF-16 range 8..20 of 25]");
+ assert_eq!(p.text(), Some(value));
+ }
+
+ #[test]
+ fn visible_text_observation_rejects_invalid_native_ranges() {
+ for range in [[-1, 1], [0, -1], [2, 1], [1, 1], [0, 20], [isize::MAX, isize::MAX]] {
+ let value = "a😀bc";
+ let p = super::Props {
+ role: "AXTextArea".into(),
+ value: Some(value.into()),
+ visible_text: crate::text::visible_text(value, range, super::MAX_TEXT),
+ ..Default::default()
+ };
+ assert_eq!(p.describe().0, "TextArea = \"a😀bc\"");
+ }
+ }
+
+ #[test]
+ fn visible_text_observation_bounds_long_excerpts() {
+ let value = format!("prefix{}tail", "😀".repeat(1001));
+ let p = super::Props {
+ role: "AXTextArea".into(),
+ value: Some(value.clone()),
+ visible_text: crate::text::visible_text(&value, [6, 2002], super::MAX_TEXT),
+ ..Default::default()
+ };
+ assert_eq!(
+ p.describe().0,
+ format!("TextArea = \"{}\" [visible UTF-16 range 6..2008 of 2012; shown 6..2006, truncated]", "😀".repeat(1000))
+ );
+ let empty = crate::text::visible_text(&value, [2012, 0], super::MAX_TEXT).unwrap();
+ assert!(empty.text.is_empty());
+ assert_eq!((empty.start, empty.end, empty.total), (2012, 2012, 2012));
+ }
+ use super::*;
+
+ fn r(x: f64, y: f64, w: f64, h: f64) -> Rect {
+ Rect { x, y, w, h }
+ }
+
+ const VIEW: Rect = Rect { x: 0.0, y: 100.0, w: 800.0, h: 500.0 };
+
+ fn shown(x: f64, y: f64, w: f64, h: f64) -> bool {
+ visible(r(x, y, w, h), VIEW).is_some()
+ }
+
+ #[test]
+ fn primary_click_action_is_implicit_but_keeps_unnamed_controls() {
+ let mut p = Props { role: "AXButton".into(), actions: vec!["AXPress".into()], ..Props::default() };
+ assert_eq!(p.describe(), ("Button".into(), true));
+ p.actions.push("AXShowMenu".into());
+ assert_eq!(p.describe(), ("Button <ShowMenu>".into(), true));
+ }
+
+ #[test]
+ fn finds_the_visible_part_of_a_frame() {
+ assert!(shown(10.0, 120.0, 50.0, 20.0));
+ let part = visible(r(10.0, 590.0, 50.0, 40.0), VIEW).unwrap();
+ assert_eq!((part.y, part.h), (590.0, 10.0), "partly visible");
+ assert!(!shown(10.0, 700.0, 50.0, 20.0));
+ assert!(!shown(10.0, 20.0, 50.0, 20.0));
+ assert!(!shown(900.0, 120.0, 50.0, 20.0));
+ }
+
+ #[test]
+ fn zero_size_frames_on_the_viewport_edge_are_offscreen() {
+ assert!(!shown(10.0, 600.0, 120.0, 0.0), "Chromium below the fold");
+ assert!(!shown(10.0, 100.0, 120.0, 0.0), "Chromium above the fold");
+ assert_eq!(visible(r(10.0, 300.0, 0.0, 0.0), VIEW).map(|v| (v.x, v.y)), Some((10.0, 300.0)), "layout wrapper inside the view");
+ assert!(shown(0.0, 300.0, 800.0, 0.0), "zero-height wrapper at left edge");
+ }
+
+ #[test]
+ fn intersects_rectangles() {
+ let i = intersect(r(0.0, 0.0, 100.0, 100.0), r(50.0, 50.0, 100.0, 100.0)).unwrap();
+ assert_eq!((i.x, i.y, i.w, i.h), (50.0, 50.0, 50.0, 50.0));
+ assert!(intersect(r(0.0, 0.0, 10.0, 10.0), r(10.0, 0.0, 10.0, 10.0)).is_none(), "touching edges");
+ }
+
+ fn lines(items: &[(Key, &str)]) -> Vec<(Key, String)> {
+ items.iter().map(|(k, l)| (*k, l.to_string())).collect()
+ }
+
+ #[test]
+ fn renders_the_whole_tree_without_a_base() {
+ let out = render(&lines(&[(Key::Element(0), "[0] Button \"OK\"")]), None, &HashSet::new());
+ assert_eq!(out.tree, "[0] Button \"OK\"\n");
+ assert!(!out.diffed);
+ }
+
+ #[test]
+ fn marks_added_and_changed_lines_and_lists_removed_ids() {
+ let before = lines(&[
+ (Key::Element(0), "[0] A"),
+ (Key::Element(1), "[1] B"),
+ (Key::Element(2), "[2] C"),
+ (Key::Element(3), "[3] D"),
+ (Key::Element(4), "[4] E"),
+ (Key::Note(ROOT, 0), "… 3 offscreen"),
+ ]);
+ let base: HashMap<Key, String> = before.into_iter().collect();
+ let after = lines(&[
+ (Key::Element(0), "[0] A"),
+ (Key::Element(1), "[1] B changed"),
+ (Key::Element(2), "[2] C"),
+ (Key::Element(4), "[4] E"),
+ (Key::Element(5), " [5] F"),
+ (Key::Note(ROOT, 0), "… 3 offscreen"),
+ ]);
+ let out = render(&after, Some(&base), &HashSet::new());
+ assert!(out.diffed);
+ assert_eq!(out.tree, "~ [1] B changed\n+ [5] F\n");
+ assert_eq!(out.removed, vec![3]);
+ }
+
+ #[test]
+ fn falls_back_to_the_whole_tree_when_most_lines_changed() {
+ let base: HashMap<Key, String> = lines(&[(Key::Element(0), "[0] A"), (Key::Element(1), "[1] B")]).into_iter().collect();
+ let after = lines(&[(Key::Element(0), "[0] A2"), (Key::Element(1), "[1] B2"), (Key::Element(2), "[2] C")]);
+ let out = render(&after, Some(&base), &HashSet::new());
+ assert!(!out.diffed);
+ assert_eq!(out.tree, "[0] A2\n[1] B2\n[2] C\n");
+ assert!(out.removed.is_empty());
+ }
+
+ #[test]
+ fn removed_selection_and_truncation_notes_refresh_the_tree() {
+ let after = lines(&[(Key::Element(0), "[0] TextField"), (Key::Info(0), "Focus: [0]")]);
+ for (key, text) in [(Key::Info(1), "Selected text: abc"), (Key::Note(ROOT, 0), "… 3 offscreen")] {
+ let mut base: HashMap<Key, String> = after.iter().cloned().collect();
+ base.insert(key, text.into());
+ let out = render(&after, Some(&base), &HashSet::new());
+ assert!(!out.diffed);
+ assert_eq!(out.tree, "[0] TextField\nFocus: [0]\n");
+ }
+ }
+
+ #[test]
+ fn reports_no_change_as_an_empty_diff() {
+ let after = lines(&[(Key::Element(0), "[0] A")]);
+ let base: HashMap<Key, String> = after.iter().cloned().collect();
+ let out = render(&after, Some(&base), &HashSet::new());
+ assert!(out.diffed);
+ assert_eq!(out.tree, "");
+ }
+
+ fn item(id: u32, text: &str, kids: Vec<Item>) -> Item {
+ Item { key: Key::Element(id), text: text.into(), kids }
+ }
+
+ fn texts(out: &[(Key, String)]) -> Vec<&str> {
+ out.iter().map(|(_, l)| l.as_str()).collect()
+ }
+
+ #[test]
+ fn lays_out_everything_within_budget() {
+ let tree = vec![item(0, "A", vec![item(1, "B", vec![])]), item(2, "C", vec![])];
+ let (out, cut) = layout(&tree, 1000);
+ assert!(!cut);
+ assert_eq!(texts(&out), vec!["[0] A", " [1] B", "[2] C"]);
+ }
+
+ #[test]
+ fn shares_the_budget_between_siblings() {
+ let rows = |base: u32| (0..200).map(|i| item(base + i, &format!("Row {}", "x".repeat(20)), vec![])).collect();
+ let tree = vec![item(1000, "Sidebar", rows(0)), item(2000, "Content", rows(300)), item(3000, "Footer", vec![])];
+ let (out, cut) = layout(&tree, 4000);
+ assert!(cut);
+ let total: usize = out.iter().map(|(_, l)| l.len() + 1).sum();
+ assert!(total <= 4000 + CUT_NOTE, "{total}");
+ let lines = texts(&out);
+ assert!(lines.contains(&"[3000] Footer"), "a later sibling is not starved");
+ let count = |r: std::ops::Range<u32>| out.iter().filter(|(k, _)| matches!(k, Key::Element(i) if r.contains(i))).count();
+ let (sidebar, content) = (count(0..200), count(300..500));
+ assert!(sidebar > 20 && content > 20, "{sidebar} {content}");
+ assert!(out
+ .iter()
+ .any(|(k, l)| *k == Key::Note(1000, u32::MAX) && l.ends_with("more not shown; the observation size limit was reached")));
+ assert!(out.iter().any(|(k, _)| *k == Key::Note(2000, u32::MAX)));
+ }
+
+ #[test]
+ fn stays_within_budget_on_wide_deep_trees() {
+ let tree: Vec<Item> = (0..300)
+ .map(|i| {
+ let leaves: Vec<Item> = (0..20)
+ .map(|j| {
+ let mut deep: Vec<Item> =
+ (0..20).map(|k| item(100_000 + i * 400 + j * 20 + k, &"z".repeat(30 + k as usize), vec![])).collect();
+ deep.push(Item::leaf(Key::Note(10_000 + i * 20 + j, 0), "… 12345 more offscreen; scroll to reveal".into()));
+ Item { key: Key::Element(10_000 + i * 20 + j), text: "Row".into(), kids: deep }
+ })
+ .collect();
+ Item { key: Key::Element(i), text: format!("Section {i}"), kids: leaves }
+ })
+ .collect();
+ for budget in [300, 2_000, 20_000, 60_000] {
+ let (out, cut) = layout(&tree, budget);
+ let total: usize = out.iter().map(|(_, l)| l.len() + 1).sum();
+ assert!(cut);
+ assert!(total <= budget, "budget {budget}: {total}");
+ assert!(out.iter().any(|(k, _)| matches!(k, Key::Note(_, u32::MAX))));
+ }
+ }
+
+ #[test]
+ fn keeps_a_prefix_when_even_single_lines_do_not_fit() {
+ let tree: Vec<Item> = (0..50).map(|i| item(i, &"y".repeat(40), vec![])).collect();
+ let (out, cut) = layout(&tree, 500);
+ assert!(cut);
+ assert_eq!(out.first().unwrap().0, Key::Element(0));
+ let (key, last) = out.last().unwrap();
+ assert_eq!(*key, Key::Note(ROOT, u32::MAX));
+ assert!(last.starts_with(&format!("… {} more", 50 - (out.len() - 1))));
+ }
+
+ #[test]
+ fn writes_headings_as_markdown() {
+ let mut h = props("AXHeading");
+ h.title = Some("Getting started".into());
+ h.value = Some("2".into());
+ assert_eq!(h.describe().0, "## Getting started");
+ h.actions = vec!["AXPress".into()];
+ assert_eq!(h.describe().0, "## Getting started");
+ let mut bare = props("AXHeading");
+ bare.value = Some("2".into());
+ assert!(bare.folds_text());
+ assert_eq!(bare.describe(), ("Heading".into(), false), "no text: folded away");
+ }
+
+ #[test]
+ fn writes_web_list_items_as_bullets() {
+ let mut list = props("AXList");
+ list.subrole = Some("AXContentList".into());
+ assert!(list.content_list());
+ let mut item = props("AXGroup");
+ item.list_item = true;
+ assert!(item.folds_text());
+ assert_eq!(item.describe(), ("-".into(), true), "item with structured children");
+ item.title = Some("Milk".into());
+ assert_eq!(item.describe().0, "- Milk");
+ let mut link = props("AXLink");
+ link.list_item = true;
+ link.title = Some("Docs".into());
+ assert_eq!(link.describe().0, "Link \"Docs\"", "non-group items keep their role");
+ }
+
+ fn props(role: &str) -> Props {
+ Props { role: role.into(), ..Default::default() }
+ }
+
+ #[test]
+ fn describes_richer_element_state() {
+ let mut field = props("AXTextField");
+ field.subrole = Some("AXSearchField".into());
+ field.label = Some("Search mail".into());
+ field.placeholder = Some("Search".into());
+ field.settable = true;
+ field.focused = true;
+ assert_eq!(field.describe(), ("TextField:SearchField \"Search mail\" placeholder \"Search\" [focused settable]".into(), true));
+
+ let mut link = props("AXLink");
+ link.title = Some("Docs".into());
+ link.url = Some("https://example.com/docs".into());
+ link.actions = vec!["AXPress".into()];
+ assert_eq!(link.describe().0, "Link \"Docs\" url \"https://example.com/docs\"");
+
+ let mut check = props("AXCheckBox");
+ check.title = Some("Remember me".into());
+ check.value = Some("1".into());
+ assert_eq!(check.describe().0, "CheckBox \"Remember me\" [checked]");
+ check.value = Some("0".into());
+ assert_eq!(check.describe().0, "CheckBox \"Remember me\" [unchecked]");
+
+ let mut disclosure = props("AXDisclosureTriangle");
+ disclosure.expanded = Some(false);
+ disclosure.disabled = true;
+ assert_eq!(disclosure.describe(), ("DisclosureTriangle [disabled collapsed]".into(), true));
+
+ let mut window = props("AXWindow");
+ window.subrole = Some("AXStandardWindow".into());
+ assert_eq!(window.describe(), ("Window".into(), false), "unremarkable containers fold away");
+ }
+
+ #[test]
+ fn keeps_long_text_values() {
+ let mut text = props("AXStaticText");
+ text.value = Some("x".repeat(500));
+ assert_eq!(text.describe().0.len(), "StaticText = \"\"".len() + 500);
+ let mut button = props("AXPopUpButton");
+ button.value = Some("y".repeat(500));
+ assert!(button.describe().0.ends_with("…\""), "short limit for non-text roles");
+ }
+
+ #[test]
+ fn placeholder_only_shows_when_empty() {
+ let mut field = props("AXTextField");
+ field.value = Some("hello".into());
+ field.placeholder = Some("Type here".into());
+ assert_eq!(field.describe().0, "TextField = \"hello\"");
+ }
+
+ #[test]
+ fn merges_neighbouring_text_runs() {
+ let line = |y: f64| Some(r(0.0, y, 50.0, 20.0));
+ let parts: Vec<(String, Option<Rect>)> =
+ [("Read the ", 0.0), ("docs", 0.0), (", then ", 0.0), ("Hel", 0.0), ("lo", 0.0), ("Next line", 20.0)]
+ .iter()
+ .map(|(s, y)| (s.to_string(), line(*y)))
+ .collect();
+ assert_eq!(merged_text(&parts), "StaticText = \"Read the docs, then Hello⏎Next line\"");
+ let mut t = props("AXStaticText");
+ t.value = Some("hi".into());
+ assert!(t.plain_text());
+ t.actions = vec!["AXPress".into()];
+ assert!(!t.plain_text(), "actionable text stays its own line");
+ }
+
+ fn loc(name: &str) -> Locator {
+ let mut p = props("AXButton");
+ p.title = Some(name.into());
+ Locator::ROOT.child(&p)
+ }
+
+ #[test]
+ fn locators_follow_role_name_and_ancestry() {
+ assert_eq!(loc("OK"), loc("OK"));
+ assert_ne!(loc("OK"), loc("Cancel"));
+ let mut group = props("AXGroup");
+ group.identifier = Some("toolbar".into());
+ let mut ok = props("AXButton");
+ ok.title = Some("OK".into());
+ assert_ne!(Locator::ROOT.child(&group).child(&ok), loc("OK"));
+ }
+
+ #[test]
+ fn ids_survive_offscreen_pruning() {
+ let mut ids: Ids<&str> = Ids::new();
+ let a = ids.assign(&"a", 1, loc("a"));
+ let b = ids.assign(&"b", 2, loc("b"));
+ let mut ids = ids.carry();
+ assert_eq!(ids.assign(&"a", 1, loc("a")), a);
+ let mut ids = ids.carry();
+ let c = ids.assign(&"c", 3, loc("c"));
+ let mut ids = ids.carry();
+ assert_eq!(ids.assign(&"b", 2, loc("b")), b, "scrolled back: same id");
+ assert_eq!(ids.assign(&"c", 3, loc("c")), c);
+ assert_eq!(ids.next, 3);
+ assert!(ids.seen(&"b", 2) && !ids.seen(&"a", 1), "only this walk counts as seen");
+ }
+
+ #[test]
+ fn parked_ids_expire() {
+ let mut ids: Ids<&str> = Ids::new();
+ ids.assign(&"a", 1, loc("a"));
+ for _ in 0..=PARK_OBSERVATIONS + 1 {
+ ids = ids.carry();
+ }
+ assert_eq!(ids.assign(&"a", 1, loc("a")), 1, "forgotten after too long");
+ }
+
+ #[test]
+ fn rebuilt_elements_inherit_ids_by_locator() {
+ let mut ids: Ids<&str> = Ids::new();
+ let send = ids.assign(&"send-v1", 10, loc("Send"));
+ assert!(ids.rebind(|_| true).is_empty(), "nothing to inherit from on the first walk");
+ let mut ids = ids.carry();
+ let fresh = ids.assign(&"send-v2", 11, loc("Send"));
+ assert_eq!(ids.rebind(|e| *e == "send-v1"), HashMap::from([(fresh, send)]), "old object is dead");
+ let mut ids = ids.carry();
+ assert_eq!(ids.assign(&"send-v2", 11, loc("Send")), send, "the new object is what is remembered");
+ let mut ids = ids.carry();
+ ids.assign(&"send-v3", 12, loc("Send"));
+ assert!(ids.rebind(|_| false).is_empty(), "old object still alive: a second button");
+ }
+
+ #[test]
+ fn ambiguous_locators_get_new_ids() {
+ let mut ids: Ids<&str> = Ids::new();
+ let row1 = ids.assign(&"row1", 1, loc("Row"));
+ ids.assign(&"row2", 2, loc("Row"));
+ let mut ids = ids.carry();
+ ids.assign(&"row3", 3, loc("Row"));
+ assert!(ids.rebind(|_| true).is_empty(), "two old candidates");
+ let mut ids = ids.carry();
+ assert_eq!(ids.assign(&"row1", 1, loc("Row")), row1, "same object still matches directly");
+ }
+
+ #[test]
+ fn one_old_two_new_is_ambiguous() {
+ let mut ids: Ids<&str> = Ids::new();
+ let send = ids.assign(&"send", 1, loc("Send"));
+ let mut ids = ids.carry();
+ let a = ids.assign(&"send-a", 2, loc("Send"));
+ let b = ids.assign(&"send-b", 3, loc("Send"));
+ assert!(ids.rebind(|_| true).is_empty());
+ assert!(a != send && b != send);
+ }
+
+ #[test]
+ fn live_locator_validates_the_observed_child_path() {
+ let group = Props { role: "AXGroup".into(), identifier: Some("toolbar".into()), ..Default::default() };
+ let button = Props { role: "AXButton".into(), title: Some("Send".into()), ..Default::default() };
+ let mut nodes = HashMap::from([(0, (Props::default(), vec![1])), (1, (group.clone(), vec![2, 99])), (2, (button.clone(), vec![]))]);
+ let roots = [(1, Locator::ROOT)];
+ let ancestry = [0, 1, 2];
+ let expected = Locator::ROOT.child(&group).child(&button);
+ assert_eq!(path_locator(&2, &roots, 10, &ancestry, |id| nodes.get(id).cloned()), Some(expected));
+ assert_eq!(path_locator(&2, &roots, 10, &[1, 2], |id| nodes.get(id).cloned()), Some(expected));
+ nodes.get_mut(&2).unwrap().0.title = Some("Delete".into());
+ assert_ne!(path_locator(&2, &roots, 10, &ancestry, |id| nodes.get(id).cloned()), Some(expected));
+ nodes.get_mut(&2).unwrap().0 = button;
+ nodes.get_mut(&1).unwrap().1 = vec![99];
+ assert_eq!(path_locator(&2, &roots, 10, &ancestry, |id| nodes.get(id).cloned()), None);
+ nodes.get_mut(&1).unwrap().1 = vec![2];
+ assert_eq!(path_locator(&2, &[], 10, &ancestry, |id| nodes.get(id).cloned()), None);
+ assert_eq!(path_locator(&2, &roots, 0, &ancestry, |id| nodes.get(id).cloned()), None);
+ nodes.get_mut(&1).unwrap().1.push(1);
+ assert_eq!(path_locator(&2, &roots, 10, &[0, 1, 1, 2], |id| nodes.get(id).cloned()), None);
+ nodes.get_mut(&0).unwrap().1.clear();
+ assert_eq!(path_locator(&2, &roots, 10, &ancestry, |id| nodes.get(id).cloned()), None);
+ }
+
+ #[test]
+ fn live_locator_rejects_identical_container_reparenting() {
+ let group = Props { role: "AXGroup".into(), ..Default::default() };
+ let button = Props { role: "AXButton".into(), title: Some("Delete".into()), ..Default::default() };
+ let mut nodes = HashMap::from([
+ (0, (Props::default(), vec![1, 2])),
+ (1, (group.clone(), vec![3])),
+ (2, (group.clone(), vec![])),
+ (3, (button.clone(), vec![])),
+ ]);
+ let roots = [(1, Locator::ROOT), (2, Locator::ROOT)];
+ let expected = Locator::ROOT.child(&group).child(&button);
+ let ancestry = [0, 1, 3];
+ assert_eq!(path_locator(&3, &roots, 10, &ancestry, |id| nodes.get(id).cloned()), Some(expected));
+ nodes.get_mut(&1).unwrap().1.clear();
+ nodes.get_mut(&2).unwrap().1.push(3);
+ assert_eq!(path_locator(&3, &roots, 10, &[0, 2, 3], |id| nodes.get(id).cloned()), Some(expected));
+ assert_eq!(path_locator(&3, &roots, 10, &ancestry, |id| nodes.get(id).cloned()), None);
+ }
+
+ #[test]
+ fn reused_objects_get_new_ids_when_their_identity_changes() {
+ let mut ids: Ids<&str> = Ids::new();
+ let original = ids.assign(&"cell", 1, loc("Alice"));
+ let mut ids = ids.carry();
+ let recycled = ids.assign(&"cell", 1, loc("Bob"));
+ assert_ne!(recycled, original);
+ assert_eq!(ids.locator(recycled), Some(loc("Bob")));
+ let mut ids = ids.carry();
+ let parent = Locator::ROOT.child(&props("AXToolbar"));
+ let reparented = parent.child(&Props { role: "AXButton".into(), title: Some("Bob".into()), ..Default::default() });
+ assert_ne!(ids.assign(&"cell", 1, reparented), recycled);
+ }
+
+ #[test]
+ fn surviving_duplicate_prevents_rebinding_a_new_object() {
+ let mut ids: Ids<&str> = Ids::new();
+ let old = ids.assign(&"old", 1, loc("Send"));
+ ids.assign(&"survivor", 2, loc("Send"));
+ let mut ids = ids.carry();
+ ids.assign(&"survivor", 2, loc("Send"));
+ let new = ids.assign(&"new", 3, loc("Send"));
+ assert_ne!(new, old);
+ assert!(ids.rebind(|e| *e == "old").is_empty());
+ }
+
+ #[test]
+ fn action_relocation_rejects_a_surviving_duplicate_or_existing_owner() {
+ let mut ids = Ids::new();
+ let removed = ids.assign(&"removed", 1, loc("Delete"));
+ ids.assign(&"survivor", 2, loc("Delete"));
+ let ids = ids.carry();
+ assert_eq!(ids.relocate(removed, |_| Some("survivor")), None);
+ let mut ids = Ids::new();
+ let removed = ids.assign(&"removed", 1, loc("Delete"));
+ ids.assign(&"renamed", 2, loc("Cancel"));
+ let ids = ids.carry();
+ assert_eq!(ids.relocate(removed, |_| Some("renamed")), None);
+ assert_eq!(ids.relocate(removed, |_| Some("replacement")), Some("replacement"));
+ }
+
+ #[test]
+ fn observation_rebinding_does_not_take_another_ids_native_object() {
+ let mut ids = Ids::new();
+ let removed = ids.assign(&"removed", 1, loc("Delete"));
+ let other = ids.assign(&"renamed", 2, loc("Cancel"));
+ let mut ids = ids.carry();
+ let changed = ids.assign(&"renamed", 2, loc("Delete"));
+ assert!(ids.rebind(|element| *element == "removed").is_empty());
+ assert_ne!(changed, removed);
+ assert_ne!(changed, other);
+ }
+
+ #[test]
+ fn collection_items_never_inherit_ids() {
+ let mut row = props("AXRow");
+ row.identifier = None;
+ let mut reply = props("AXButton");
+ reply.title = Some("Reply".into());
+ let inside = Locator::ROOT.child(&props("AXTable")).child(&row).child(&reply);
+ assert!(!inside.rebindable);
+ let mut ids: Ids<&str> = Ids::new();
+ ids.assign(&"reply-row1", 1, inside);
+ let mut ids = ids.carry();
+ ids.assign(&"reply-row11", 2, inside);
+ assert!(ids.rebind(|_| true).is_empty());
+ }
+
+ #[test]
+ fn rebound_lines_count_as_changed() {
+ let base: HashMap<Key, String> =
+ lines(&[(Key::Element(0), "[0] A"), (Key::Element(1), "[1] B"), (Key::Element(2), "[2] C")]).into_iter().collect();
+ let after = lines(&[(Key::Element(0), "[0] A"), (Key::Element(1), "[1] B"), (Key::Element(2), "[2] C")]);
+ let out = render(&after, Some(&base), &HashSet::from([1]));
+ assert_eq!(out.tree, "~ [1] B\n");
+ }
+
+ #[test]
+ fn spots_large_regions_with_nothing_readable() {
+ let window = r(0.0, 0.0, 1000.0, 800.0);
+ assert!(sparse(r(0.0, 80.0, 1000.0, 700.0), window, 0));
+ assert!(!sparse(r(0.0, 80.0, 1000.0, 700.0), window, 3));
+ assert!(!sparse(r(0.0, 0.0, 100.0, 100.0), window, 0));
+ }
+}
diff --git a/desktop/desk/tests/cancellation.rs b/desktop/desk/tests/cancellation.rs
new file mode 100644
index 000000000..1d24dfac7
--- /dev/null
+++ b/desktop/desk/tests/cancellation.rs
@@ -0,0 +1,31 @@
+#![cfg(target_os = "macos")]
+
+use std::io::{BufRead, BufReader, Write};
+use std::process::{Command, Stdio};
+use std::time::{Duration, Instant};
+
+#[test]
+fn serve_exits_on_sigterm_with_stdin_still_open() {
+ let mut child = Command::new(env!("CARGO_BIN_EXE_desk")).arg("serve").stdin(Stdio::piped()).stdout(Stdio::piped()).spawn().unwrap();
+ let mut input = child.stdin.take().unwrap();
+ writeln!(input, "invalid JSON").unwrap();
+ let mut output = BufReader::new(child.stdout.take().unwrap());
+ let mut line = String::new();
+ output.read_line(&mut line).unwrap();
+ assert!(line.contains("bad_request"), "{line}");
+ assert_eq!(unsafe { libc::kill(child.id() as i32, libc::SIGTERM) }, 0);
+ let deadline = Instant::now() + Duration::from_secs(2);
+ loop {
+ if let Some(status) = child.try_wait().unwrap() {
+ assert!(status.success(), "{status}");
+ break;
+ }
+ if Instant::now() >= deadline {
+ child.kill().unwrap();
+ child.wait().unwrap();
+ panic!("serve did not exit after SIGTERM while stdin remained open");
+ }
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ drop(input);
+}
diff --git a/desktop/desk/tests/manual/FOCUS-TRACE.md b/desktop/desk/tests/manual/FOCUS-TRACE.md
new file mode 100644
index 000000000..cd68ce532
--- /dev/null
+++ b/desktop/desk/tests/manual/FOCUS-TRACE.md
@@ -0,0 +1,5 @@
+Compile on a Mac with `clang -fobjc-arc -framework ApplicationServices -framework CoreGraphics -framework Foundation tests/manual/focus-trace.m -o /tmp/desk-focus-trace` from `desktop/desk`.
+
+Run `/tmp/desk-focus-trace 2> /tmp/desk-focus-trace.log`, then run `python3 tests/manual/background-input.py --desk target/release/desk --app com.google.Chrome --x FIELD_X --y FIELD_Y` against an empty disposable field while it observes for 20 seconds. The trace logs only monotonic time, source and target process IDs, notification subtype, focus subject PID, theft ID, live AX foreground PID, and the first eight window IDs before/after. It does not inspect or suppress keyboard text. Check whether foreground PID and window order remain stable, and whether type-21 notifications appear around synthetic input. This is a diagnostic, not proof of prevention; do not post its log publicly without reviewing it.
+
+For the lease protocol, correlate `0x4000` (taken), `0xf102` (changed, carrying the theft ID), and `0x8000` (returned). A `0x2` notification is a new-front transition, not the same as a temporary key-focus lease. Record the macOS version and candidate commit. The trace is diagnostic instrumentation and can affect timing; repeat acceptance checks without it.
diff --git a/desktop/desk/tests/manual/background-input.py b/desktop/desk/tests/manual/background-input.py
new file mode 100644
index 000000000..6c09d213a
--- /dev/null
+++ b/desktop/desk/tests/manual/background-input.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+import argparse
+import json
+from pathlib import Path
+import select
+import subprocess
+import sys
+import tempfile
+import time
+import uuid
+
+
+def receive(process, timeout=15):
+ readable, _, _ = select.select([process.stdout], [], [], timeout)
+ if not readable:
+ raise RuntimeError("desk response timed out")
+ line = process.stdout.readline()
+ if not line:
+ raise RuntimeError("desk exited before replying")
+ response = json.loads(line)["reply"]
+ if response["type"] == "error":
+ raise RuntimeError(response.get("message", "desk error"))
+ return response
+
+
+def request(process, command, sequence):
+ process.stdin.write(json.dumps({"id": sequence, "cmd": command}) + "\n")
+ process.stdin.flush()
+ return receive(process)
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Manual no-focus-steal acceptance test; writes only a random marker to a disposable field.")
+ parser.add_argument("--desk", required=True, type=Path)
+ parser.add_argument("--app", required=True)
+ parser.add_argument("--x", required=True, type=float, help="Editable field x in window-relative points, not Retina pixels")
+ parser.add_argument("--y", required=True, type=float, help="Editable field y in window-relative points, not Retina pixels")
+ parser.add_argument("--witness", type=Path, help="Precompiled focus-witness executable")
+ args = parser.parse_args()
+ if sys.platform != "darwin":
+ parser.error("This test must run on a Mac with Accessibility permission for its launcher.")
+ marker = "desk-background-" + uuid.uuid4().hex
+ with tempfile.TemporaryDirectory(prefix="desk-focus-") as directory:
+ directory = Path(directory)
+ witness_path = args.witness or directory / "witness"
+ if not args.witness:
+ subprocess.run(["swiftc", str(Path(__file__).with_name("focus-witness.swift")), "-o", str(witness_path)], check=True)
+ desk = subprocess.Popen([str(args.desk.resolve()), "serve"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
+ witness = None
+ stop = directory / "stop"
+ try:
+ observed = request(desk, {"type": "observe", "app": args.app, "screenshot": False, "diff": False}, 1)
+ target_pid = observed["pid"]
+ print("Use an empty disposable target field. Switch to another app now and type a harmless test sentence there; do not move the mouse during the test.", flush=True)
+ time.sleep(7)
+ ready = directory / "ready"
+ witness = subprocess.Popen([str(witness_path.resolve()), str(target_pid), str(ready), str(stop)], stdout=subprocess.PIPE, text=True)
+ deadline = time.monotonic() + 5
+ while not ready.exists():
+ if witness.poll() is not None or time.monotonic() >= deadline:
+ raise RuntimeError("Witness could not start; target must not be frontmost.")
+ time.sleep(0.02)
+ result = request(desk, {"type": "act", "app": args.app, "actions": [
+ {"type": "click", "target": {"point": {"x": args.x, "y": args.y}}},
+ {"type": "type_text", "text": marker},
+ {"type": "paste", "text": "-pasted"}
+ ], "pace": False, "observe": True, "screenshot": False, "diff": False}, 2)
+ desk.stdin.close()
+ desk.wait(timeout=2)
+ time.sleep(0.5)
+ stop.touch()
+ witness_result = json.loads(witness.communicate(timeout=5)[0])
+ failure = result.get("failure") or result.get("observation_failure")
+ accepted = marker + "-pasted" in (result.get("observation") or {}).get("tree", "")
+ passed = not failure and accepted and not witness_result["violations"]
+ print(json.dumps({"passed": passed, "markerObservedInTarget": accepted,
+ "violations": witness_result["violations"], "actionFailure": failure}, indent=2))
+ print("Also verify your sentence stayed intact in the foreground app. This sampler cannot prove that no sub-10ms window-order change occurred.")
+ return 0 if passed else 1
+ finally:
+ stop.touch()
+ if witness and witness.poll() is None:
+ try:
+ witness.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ witness.terminate()
+ witness.wait(timeout=5)
+ if not desk.stdin.closed:
+ desk.stdin.close()
+ desk.terminate()
+ try:
+ desk.wait(timeout=2)
+ except subprocess.TimeoutExpired:
+ desk.kill()
+ desk.wait()
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/desktop/desk/tests/manual/focus-trace.m b/desktop/desk/tests/manual/focus-trace.m
new file mode 100644
index 000000000..87f00efe6
--- /dev/null
+++ b/desktop/desk/tests/manual/focus-trace.m
@@ -0,0 +1,78 @@
+#import <ApplicationServices/ApplicationServices.h>
+#import <CoreGraphics/CoreGraphics.h>
+#import <Foundation/Foundation.h>
+#include <stdio.h>
+#include <time.h>
+
+static double elapsed(void) {
+ struct timespec now;
+ clock_gettime(CLOCK_MONOTONIC, &now);
+ return now.tv_sec + now.tv_nsec / 1e9;
+}
+
+static pid_t focused_pid(void) {
+ AXUIElementRef system = AXUIElementCreateSystemWide();
+ AXUIElementRef app = NULL;
+ pid_t pid = 0;
+ if (AXUIElementCopyAttributeValue(system, kAXFocusedApplicationAttribute, (CFTypeRef *)&app) == kAXErrorSuccess && app) {
+ AXUIElementGetPid(app, &pid);
+ CFRelease(app);
+ }
+ CFRelease(system);
+ return pid;
+}
+
+static CGEventRef witness(CGEventTapProxy proxy, CGEventType type, CGEventRef event, void *context) {
+ (void)proxy;
+ (void)context;
+ if ((uint32_t)type == 21) {
+ fprintf(stderr, "%.6f process-notification source=%lld target=%lld subtype=0x%llx subject=%lld theft=%lld foreground=%d\n",
+ elapsed(),
+ CGEventGetIntegerValueField(event, kCGEventSourceUnixProcessID),
+ CGEventGetIntegerValueField(event, kCGEventTargetUnixProcessID),
+ (unsigned long long)CGEventGetIntegerValueField(event, (CGEventField)64),
+ CGEventGetIntegerValueField(event, (CGEventField)73),
+ CGEventGetIntegerValueField(event, (CGEventField)71),
+ focused_pid());
+ }
+ return event;
+}
+
+static void order_snapshot(const char *label) {
+ CFArrayRef windows = CGWindowListCopyWindowInfo(kCGWindowListOptionOnScreenOnly, kCGNullWindowID);
+ fprintf(stderr, "%.6f %s foreground=%d windows=", elapsed(), label, focused_pid());
+ if (windows) {
+ CFIndex count = CFArrayGetCount(windows);
+ for (CFIndex i = 0; i < count && i < 8; i++) {
+ CFDictionaryRef info = CFArrayGetValueAtIndex(windows, i);
+ CFNumberRef number = CFDictionaryGetValue(info, kCGWindowNumber);
+ int window = 0;
+ if (number && CFNumberGetValue(number, kCFNumberIntType, &window)) fprintf(stderr, "%d,", window);
+ }
+ CFRelease(windows);
+ }
+ fputc('\n', stderr);
+}
+
+int main(void) {
+ @autoreleasepool {
+ CGEventMask mask = (CGEventMask)1 << 21;
+ CFMachPortRef tap = CGEventTapCreate(kCGAnnotatedSessionEventTap, kCGHeadInsertEventTap,
+ kCGEventTapOptionListenOnly, mask, witness, NULL);
+ if (!tap) {
+ fputs("Cannot install listen-only event tap; check Accessibility/Input Monitoring permissions.\n", stderr);
+ return 1;
+ }
+ CFRunLoopSourceRef source = CFMachPortCreateRunLoopSource(NULL, tap, 0);
+ if (!source) { CFRelease(tap); return 1; }
+ CFRunLoopAddSource(CFRunLoopGetCurrent(), source, kCFRunLoopDefaultMode);
+ order_snapshot("start");
+ double end = elapsed() + 20.0;
+ while (elapsed() < end) CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.05, false);
+ order_snapshot("end");
+ CFRunLoopRemoveSource(CFRunLoopGetCurrent(), source, kCFRunLoopDefaultMode);
+ CFRelease(source);
+ CFRelease(tap);
+ }
+ return 0;
+}
diff --git a/desktop/desk/tests/manual/focus-witness.swift b/desktop/desk/tests/manual/focus-witness.swift
new file mode 100644
index 000000000..6e71e14ae
--- /dev/null
+++ b/desktop/desk/tests/manual/focus-witness.swift
@@ -0,0 +1,50 @@
+import AppKit
+import CoreGraphics
+
+let args = CommandLine.arguments
+if args.count != 4 { exit(2) }
+let target = pid_t(args[1])!
+let ready = args[2]
+let stop = args[3]
+let workspace = NSWorkspace.shared
+let foreground = workspace.frontmostApplication?.processIdentifier
+if foreground == nil || foreground == target { exit(3) }
+let pointer = CGEvent(source: nil)!.location
+var violations = Set<String>()
+
+func order() -> [UInt32] {
+ let windows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
+ return windows.compactMap { info in
+ let pid = (info[kCGWindowOwnerPID as String] as? NSNumber)?.int32Value
+ let layer = (info[kCGWindowLayer as String] as? NSNumber)?.intValue
+ guard (pid == target || pid == foreground), layer == 0 else { return nil }
+ return (info[kCGWindowNumber as String] as? NSNumber)?.uint32Value
+ }
+}
+let baseline = order()
+func check() {
+ if workspace.frontmostApplication?.processIdentifier != foreground {
+ violations.insert("foreground_changed")
+ }
+ if order() != baseline { violations.insert("window_order_changed") }
+ if CGEvent(source: nil)!.location != pointer { violations.insert("pointer_moved") }
+}
+let token = workspace.notificationCenter.addObserver(
+ forName: NSWorkspace.didActivateApplicationNotification, object: nil, queue: .main
+) { notification in
+ let app = notification.userInfo?[NSWorkspace.applicationUserInfoKey] as? NSRunningApplication
+ if app?.processIdentifier != foreground { violations.insert("foreground_changed") }
+}
+try Data("ready".utf8).write(to: URL(fileURLWithPath: ready))
+let deadline = Date().addingTimeInterval(30)
+while !FileManager.default.fileExists(atPath: stop) && Date() < deadline {
+ check()
+ RunLoop.current.run(until: Date().addingTimeInterval(0.01))
+}
+check()
+if Date() >= deadline { violations.insert("witness_timeout") }
+workspace.notificationCenter.removeObserver(token)
+let result: [String: Any] = ["violations": violations.sorted(), "sampleIntervalMs": 10]
+let data = try JSONSerialization.data(withJSONObject: result, options: [.sortedKeys])
+print(String(data: data, encoding: .utf8)!)
+exit(violations.isEmpty ? 0 : 1)
diff --git a/desktop/package-lock.json b/desktop/package-lock.json
index 91c3432ba..63defae91 100644
--- a/desktop/package-lock.json
+++ b/desktop/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "qm-desktop",
- "version": "0.1.3",
+ "version": "0.1.4",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "qm-desktop",
- "version": "0.1.3",
+ "version": "0.1.4",
"dependencies": {
"update-electron-app": "3.3.0"
},
diff --git a/desktop/package.json b/desktop/package.json
index fdef1d84f..913adc123 100644
--- a/desktop/package.json
+++ b/desktop/package.json
@@ -1,7 +1,7 @@
{
"name": "qm-desktop",
"productName": "QM",
- "version": "0.1.3",
+ "version": "0.1.4",
"private": true,
"type": "module",
"main": "main.mjs",
diff --git a/desktop/runtime.js b/desktop/runtime.js
index f67a076f9..746007412 100644
--- a/desktop/runtime.js
+++ b/desktop/runtime.js
@@ -3,6 +3,26 @@
const TEXT_LIMIT = 100_000;
const IMAGE_BUDGET = 800_000;
const IMAGE_EDGE = 1568;
+ const appNotes = new Map([
+ [
+ "com.google.Chrome",
+ "Cmd+L focuses the address bar rather than page content. Use a screenshot when a page exposes little useful accessibility text.",
+ ],
+ [
+ "com.tinyspeck.slackmacgap",
+ "Return usually sends a message; Shift+Return inserts a line break. Message and channel panes can update independently, so observe the pane you intend to edit.",
+ ],
+ [
+ "notion.id",
+ "Pages are edited as separate blocks. Click the intended block before typing; neighboring block text is not necessarily one editable value.",
+ ],
+ [
+ "com.apple.finder",
+ "Changing folders or switching view modes changes the item layout. Observe the new view before selecting file or folder targets.",
+ ],
+ ]);
+ const notedApps = new Set();
+ const observedWindows = new Map();
let emitted = new WeakSet();
let text = [];
let images = [];
@@ -47,9 +67,14 @@
async function jpeg(shot) {
const bytes = Uint8Array.from(atob(shot.png_base64), (c) => c.charCodeAt(0));
const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/png" }));
- const ratio = Math.min(1, IMAGE_EDGE / Math.max(bitmap.width, bitmap.height));
- const canvas = new OffscreenCanvas(Math.round(bitmap.width * ratio), Math.round(bitmap.height * ratio));
- canvas.getContext("2d").drawImage(bitmap, 0, 0, canvas.width, canvas.height);
+ let canvas;
+ try {
+ const ratio = Math.min(1, IMAGE_EDGE / Math.max(bitmap.width, bitmap.height));
+ canvas = new OffscreenCanvas(Math.round(bitmap.width * ratio), Math.round(bitmap.height * ratio));
+ canvas.getContext("2d").drawImage(bitmap, 0, 0, canvas.width, canvas.height);
+ } finally {
+ bitmap.close();
+ }
const blob = await canvas.convertToBlob({ type: "image/jpeg", quality: 0.8 });
const raw = new Uint8Array(await blob.arrayBuffer());
let binary = "";
@@ -57,11 +82,23 @@
return { data: btoa(binary), width: canvas.width, height: canvas.height };
}
+ function idRanges(ids) {
+ const out = [];
+ for (let i = 0; i < ids.length; i++) {
+ let j = i;
+ while (j + 1 < ids.length && ids[j + 1] === ids[j] + 1) j++;
+ out.push(j > i ? `${ids[i]}-${ids[j]}` : `${ids[i]}`);
+ i = j;
+ }
+ return out.join(", ");
+ }
+
function stateText(o) {
- const head = `${o.app} — "${o.window}" ${Math.round(o.frame.w)}x${Math.round(o.frame.h)}, ${o.elements} elements${o.truncated ? " (truncated)" : ""}`;
+ const cut = o.truncated ? " (truncated: size limit reached, some elements not shown)" : "";
+ const head = `${o.app} — "${o.window}" ${Math.round(o.frame.w)}x${Math.round(o.frame.h)}, ${o.elements} elements${cut}`;
if (!o.diffed) return `${head}\n${o.tree}`;
- const removed = o.removed.length ? `\nremoved: ${o.removed.join(", ")}` : "";
- return `${head}, changes since last state:\n${o.tree || "(no changes)\n"}${removed}`;
+ const removed = o.removed.length ? `\nremoved: ${idRanges(o.removed)}` : "";
+ return `${head}, changes since last state (+ added, ~ changed):\n${o.tree || "(no changes)\n"}${removed}`;
}
const MODIFIERS = {
@@ -76,7 +113,10 @@
};
class App {
- #scale = 1;
+ #scale = null;
+ #screenshotWindow = null;
+ #appIdentity = null;
+ #offset = { x: 0, y: 0 };
constructor(name) {
this.name = name;
@@ -84,10 +124,37 @@
async #observe(screenshot, diff, emit, tree = true) {
const o = await desk({ type: "observe", app: this.name, screenshot, diff });
+ this.#appIdentity = o.bundle_id ? `bundle:${o.bundle_id}` : `name:${o.app}`;
+ if (!Number.isInteger(o.window_id) || o.window_id <= 0 || o.window_id > 0xffffffff) {
+ observedWindows.delete(this.#appIdentity);
+ this.#scale = null;
+ this.#screenshotWindow = null;
+ throw new Error("Observation has no valid native window ID; screenshot coordinates cannot be verified.");
+ }
+ const identity = JSON.stringify([o.pid, o.window_id, o.frame.x, o.frame.y, o.frame.w, o.frame.h]);
+ observedWindows.set(this.#appIdentity, identity);
+ if (identity !== this.#screenshotWindow) this.#scale = null;
+ const note = appNotes.get(o.bundle_id);
+ if (emit && note && !notedApps.has(o.bundle_id)) {
+ const line = `App usage notes (${o.bundle_id}): ${note}`;
+ if (size + line.length <= TEXT_LIMIT) {
+ say(line);
+ notedApps.add(o.bundle_id);
+ }
+ }
if (emit && tree) say(stateText(o));
if (o.screenshot) {
+ const shot = o.screenshot.frame;
+ if (!shot || ![shot.x, shot.y, shot.w, shot.h].every(Number.isFinite) || shot.w <= 0 || shot.h <= 0) {
+ observedWindows.delete(this.#appIdentity);
+ this.#scale = null;
+ this.#screenshotWindow = null;
+ throw new Error("Observation has no valid screenshot frame; screenshot coordinates cannot be verified.");
+ }
const image = await jpeg(o.screenshot);
- this.#scale = image.width / o.frame.w;
+ this.#screenshotWindow = identity;
+ this.#scale = image.width / shot.w;
+ this.#offset = { x: shot.x - o.frame.x, y: shot.y - o.frame.y };
if (emit && images.length < 4 && imageSize + image.data.length <= IMAGE_BUDGET) {
imageSize += image.data.length;
images.push({ mimeType: "image/jpeg", data: image.data });
@@ -112,7 +179,11 @@
#target(t) {
if (typeof t === "number") return { index: t };
- if (Array.isArray(t) && t.length === 2) return { point: { x: t[0] / this.#scale, y: t[1] / this.#scale } };
+ if (Array.isArray(t) && t.length === 2) {
+ if (this.#scale === null || observedWindows.get(this.#appIdentity) !== this.#screenshotWindow)
+ throw new Error("Get a current screenshot before using point coordinates; the window may have changed.");
+ return { point: { x: this.#offset.x + t[0] / this.#scale, y: this.#offset.y + t[1] / this.#scale } };
+ }
throw new Error("target must be an element index or [x, y] screenshot pixels");
}
@@ -157,8 +228,10 @@
return this.#act({ type: "select_text", index, text, prefix, suffix, selection: selectionType });
}
- setValue(index, value) {
- return this.#act({ type: "set_value", index, value: String(value) });
+ setValue(index, value, { submit } = {}) {
+ if (submit !== undefined && typeof submit !== "boolean")
+ throw new TypeError("setValue submit must be true or false");
+ return this.#act({ type: "set_value", index, value: String(value), ...(submit === undefined ? {} : { submit }) });
}
performSecondaryAction(index, action) {
@@ -170,7 +243,11 @@
}
typeText(text) {
- return this.#act({ type: "type_text", text });
+ return this.#act({ type: "type_text", text: String(text) });
+ }
+
+ paste(text, { format = "text" } = {}) {
+ return this.#act({ type: "paste", text: String(text), format });
}
raise() {
@@ -183,6 +260,9 @@
}
window.computer = Object.freeze({
+ docs() {
+ say(window.__docs);
+ },
async listApps({ emit = true } = {}) {
const { apps } = await desk({ type: "apps" });
if (emit) say(apps.map((a) => `${a.name} (${a.bundle_id ?? "-"}) ${a.active ? " active" : ""}`).join("\n"));
@@ -197,28 +277,47 @@
});
window.__docs = `computer API (JavaScript, top-level await; bindings persist within this turn; reset with desktop {code: "", reset: true}):
+computer.docs() emits this API documentation again without resetting app objects or runtime state.
computer.listApps() -> [{name, bundle_id, pid, active}]
-computer.getApp(nameOrBundleId) -> App. Opens an installed app in the background if needed and emits its UI tree.
+computer.getApp(nameOrBundleIdOrFullPath) -> App. Opens an installed app in the background if needed and emits its UI tree.
All apps are available subject to macOS permissions.
Observation (each emits into the result; pass {emit: false} to only return the data):
-app.getAXState({disableDiffing}) the focused window's accessibility tree. Lines read
- [id] Role "title" (description) = "value" [focused disabled selected settable] <Actions> @x,y wxh
- with window-relative point frames. Ids stay stable across observations of the same window.
- By default only changed or new lines are shown, plus removed ids; disableDiffing: true shows the whole tree.
-app.getScreenshot() a JPEG of the window. app.getAXStateAndScreenshot({disableDiffing}) both.
-
-Actions (each waits about a second, longer while a progress indicator shows; observe again to see the result):
+app.getAXState({disableDiffing}) the visible part of the focused window's accessibility tree. Lines read
+ [id] Role:Subrole "name" (description) = "value" placeholder "…" url "…" [focused disabled selected checked expanded settable] <Actions>
+ Open menus and dialogs are listed first; Focus: and Selected text: lines follow the tree.
+ Content scrolled out of view is summarised as "… N more offscreen; scroll to reveal".
+ Ids stay stable across observations of the same window.
+ By default only new (+) or changed (~) lines are shown, plus removed ids; when most lines changed, or with
+ disableDiffing: true, the whole tree is shown.
+app.getScreenshot() a JPEG covering the window and its visible menus/panels. app.getAXStateAndScreenshot({disableDiffing}) both.
+ Screenshot pixel targets are translated automatically, including panels extending beyond the main window.
+
+Observe the intended app before acting and use targets from its current state. Actions return after input is sent,
+not after the requested outcome is confirmed. Observe to verify each meaningful change, and stop once the requested
+result is visible. After an error, interruption or timeout, inspect the current state before retrying an action that
+may already have completed. Observations wait for accessibility events to settle, longer while a progress indicator shows.
app.click(target, {mouseButton: left|right|middle, clickCount, key: "cmd+shift", durationMs})
target is an element id from the tree or [x, y] in pixels of the latest screenshot.
+ Ids in web content, rows and cells get a real click; other ids use the accessibility press action.
+ A real click on an id scrolls it into view first and fails if it stays offscreen or something covers it.
+ While a menu is open, click its items by id or press Escape; other clicks and pointer actions fail.
+ Input uses app-local activation rather than automatically bringing the app to the front.
+ If the foreground app changes or you interact with the target app, the action stops.
app.drag([x, y], [x, y]) screenshot pixels.
app.scroll(target | null, up|down|left|right, pages = 1 | {pixels})
-app.selectText(id, text, {prefix, suffix, selectionType: text|cursor_before|cursor_after}) prefix/suffix pick among repeats.
-app.setValue(id, value) writes the element's value directly.
+app.selectText(id, text, {prefix, suffix, selectionType: text|cursor_before|cursor_after}) Use native text or an exact observed Markdown body excerpt without tree indentation; prefix/suffix pick among repeats.
+app.setValue(id, value, {submit}) writes the element's value directly, as a number or true/false when the element holds one.
+ It fails if the element is not settable or a text field keeps its old value. Only submit: true
+ confirms the field or presses Return; omission and false never submit.
app.performSecondaryAction(id, action) runs an action the element lists, e.g. "ShowMenu", "Increment".
-app.pressKey("cmd+shift+t", {durationMs}) key names: letters, digits, punctuation, return, tab, space, delete, escape, arrows, home, end, pageup, pagedown, f1-f12.
-app.typeText(text)
-app.raise() brings the window to the front; use when an app ignores background input.
+app.pressKey("cmd+shift+t", {durationMs}) key names: letters, digits, punctuation, return, tab, space, delete, escape, arrows, home, end, pageup, pagedown, f1-f20, KP_Enter/KP_0/KP_Add and other keypad names. Modifier names support left/right forms such as Control_R.
+app.typeText(text) types with the keyboard. "\\n" presses Return and "\\t" presses Tab, so they can submit
+ a form or send a chat message; use paste for multi-line text.
+app.paste(text, {format: text|markdown|html}) pastes through the clipboard, then restores the user's clipboard once the
+ field shows the text. If it cannot confirm that, it fails and leaves the text on the clipboard instead.
+ Prefer it for long, multi-line or formatted text. Needs a focused text field.
+app.raise() explicitly brings the window to the front and takes focus.
app.wait(ms)
Await every computer operation before returning. Background callbacks persist within the turn and may run during a later call; do not schedule computer actions in background callbacks. Cancellation or timeout destroys the runtime.
diff --git a/desktop/test/runtime-coordinates.test.mjs b/desktop/test/runtime-coordinates.test.mjs
new file mode 100644
index 000000000..239754f37
--- /dev/null
+++ b/desktop/test/runtime-coordinates.test.mjs
@@ -0,0 +1,205 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+import vm from "node:vm";
+
+const source = await readFile(new URL("../runtime.js", import.meta.url), "utf8");
+
+test("coordinate actions require a screenshot of the current process, window and geometry", async () => {
+ const calls = [];
+ let screenshotFrame = false;
+ const observation = {
+ app: "Fixture",
+ pid: 100,
+ window_id: 200,
+ window: "Document",
+ frame: { x: 10, y: 20, w: 100, h: 100 },
+ elements: 1,
+ tree: "[1] Button",
+ diffed: false,
+ removed: [],
+ };
+ const window = {
+ deskHost: {
+ async request(command) {
+ calls.push(JSON.parse(JSON.stringify(command)));
+ if (command.type === "act") return { type: "acted", completed: 1 };
+ return {
+ type: "observation",
+ ...structuredClone(observation),
+ ...(command.screenshot
+ ? {
+ screenshot: {
+ png_base64: "",
+ frame:
+ screenshotFrame === false
+ ? { ...observation.frame, x: observation.frame.x - 10, w: 200 }
+ : screenshotFrame,
+ },
+ }
+ : {}),
+ };
+ },
+ },
+ };
+ vm.runInNewContext(source, {
+ window,
+ console: {},
+ Blob,
+ atob,
+ btoa,
+ createImageBitmap: async () => ({ width: 400, height: 200, close() {} }),
+ OffscreenCanvas: class {
+ constructor(width, height) {
+ this.width = width;
+ this.height = height;
+ }
+ getContext() {
+ return { drawImage() {} };
+ }
+ async convertToBlob() {
+ return new Blob(["fixture"]);
+ }
+ },
+ });
+ const app = await window.computer.getApp("Fixture", { emit: false });
+ assert.throws(() => app.click([20, 10]), /screenshot/i);
+ await app.getScreenshot({ emit: false });
+ await app.getAXState({ emit: false });
+ await app.click([20, 10]);
+ assert.deepEqual(calls.at(-1).actions[0].target.point, { x: 0, y: 5 });
+ const alias = await window.computer.getApp("/Applications/Fixture.app", { emit: false });
+ await app.click([20, 10]);
+ observation.window_id += 1;
+ await alias.getAXState({ emit: false });
+ const beforeAliasClick = calls.length;
+ assert.throws(() => app.click([20, 10]), /screenshot/i);
+ assert.equal(calls.length, beforeAliasClick);
+ await app.getScreenshot({ emit: false });
+ for (const change of [
+ () => {
+ observation.frame.w += 10;
+ },
+ () => {
+ observation.frame.x -= 20;
+ },
+ () => {
+ observation.window_id += 1;
+ },
+ () => {
+ observation.pid += 1;
+ },
+ ]) {
+ change();
+ await app.getAXState({ emit: false });
+ const before = calls.length;
+ assert.throws(() => app.click([20, 10]), /screenshot/i);
+ assert.equal(calls.length, before);
+ await app.click(1);
+ await app.getScreenshot({ emit: false });
+ await app.click([20, 10]);
+ assert.deepEqual(calls.at(-1).actions[0].target.point, { x: 0, y: 5 });
+ }
+ for (const invalidFrame of [
+ undefined,
+ null,
+ {},
+ { x: 0, y: 0, w: 0, h: 100 },
+ { x: 0, y: 0, w: 100, h: -1 },
+ { x: Infinity, y: 0, w: 100, h: 100 },
+ { x: 0, y: "0", w: 100, h: 100 },
+ ]) {
+ await alias.getScreenshot({ emit: false });
+ screenshotFrame = invalidFrame;
+ await assert.rejects(alias.getScreenshot({ emit: false }), /screenshot frame/);
+ const before = calls.length;
+ assert.throws(() => app.click([20, 10]), /screenshot/i);
+ assert.throws(() => alias.click([20, 10]), /screenshot/i);
+ assert.equal(calls.length, before);
+ screenshotFrame = false;
+ await app.getScreenshot({ emit: false });
+ await app.click([20, 10]);
+ }
+ const nativeWindowId = observation.window_id;
+ for (const invalidId of [undefined, null, 0, -1, 0.5, 0x100000000, "200"]) {
+ observation.window_id = invalidId;
+ await assert.rejects(alias.getAXState({ emit: false }), /native window ID/);
+ const before = calls.length;
+ assert.throws(() => app.click([20, 10]), /screenshot/i);
+ assert.equal(calls.length, before);
+ await assert.rejects(app.getScreenshot({ emit: false }), /native window ID/);
+ assert.throws(() => app.click([20, 10]), /screenshot/i);
+ observation.window_id = nativeWindowId;
+ await app.getScreenshot({ emit: false });
+ await app.click([20, 10]);
+ }
+});
+
+test("screenshot conversion releases decoded pixels after drawing and on drawing failure", async () => {
+ for (const fail of [false, true]) {
+ const events = [];
+ const window = {
+ deskHost: {
+ async request() {
+ return {
+ type: "observation",
+ app: "Fixture",
+ window: "Document",
+ pid: 1,
+ window_id: 2,
+ frame: { x: 0, y: 0, w: 2000, h: 1000 },
+ elements: 0,
+ tree: "",
+ diffed: false,
+ removed: [],
+ screenshot: { png_base64: "", frame: { x: 0, y: 0, w: 2000, h: 1000 } },
+ };
+ },
+ },
+ };
+ vm.runInNewContext(source, {
+ window,
+ console: {},
+ Blob,
+ atob,
+ btoa,
+ createImageBitmap: async () => ({
+ width: 2000,
+ height: 1000,
+ close() {
+ events.push("close");
+ },
+ }),
+ OffscreenCanvas: class {
+ constructor(width, height) {
+ this.width = width;
+ this.height = height;
+ }
+ getContext() {
+ return {
+ drawImage() {
+ events.push("draw");
+ if (fail) throw new Error("draw failed");
+ },
+ };
+ }
+ async convertToBlob() {
+ events.push("encode");
+ return new Blob(["fixture"]);
+ }
+ },
+ });
+ const pending = window.computer.getApp("Fixture", { emit: false });
+ if (fail) {
+ await assert.rejects(pending, /draw failed/);
+ assert.deepEqual(events, ["draw", "close"]);
+ } else {
+ const app = await pending;
+ assert.deepEqual(events, ["draw", "close", "encode"]);
+ assert.deepEqual(JSON.parse(JSON.stringify(await app.getScreenshot({ emit: false }))), {
+ width: 1568,
+ height: 784,
+ });
+ }
+ }
+});
diff --git a/desktop/test/runtime-notes.test.mjs b/desktop/test/runtime-notes.test.mjs
new file mode 100644
index 000000000..bc5ed8e3c
--- /dev/null
+++ b/desktop/test/runtime-notes.test.mjs
@@ -0,0 +1,91 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+import vm from "node:vm";
+
+const source = await readFile(new URL("../runtime.js", import.meta.url), "utf8");
+
+function runtime(onRequest = () => {}) {
+ const window = {
+ deskHost: {
+ async request(command) {
+ onRequest(command);
+ const { app } = command;
+ if (command.type === "act") return { type: "acted", completed: 1 };
+ return {
+ type: "observation",
+ app,
+ bundle_id: app,
+ window: "Test window",
+ window_id: 1,
+ frame: { w: 100, h: 100 },
+ elements: 1,
+ tree: "[1] TextField",
+ diffed: false,
+ removed: [],
+ };
+ },
+ },
+ };
+ vm.runInNewContext(source, { window, console: {} });
+ return window;
+}
+
+test("app notes emit once per bundle across App objects and output drains", async () => {
+ const window = runtime();
+ await window.computer.getApp("com.google.Chrome");
+ assert.match(window.__drain().text, /App usage notes \(com\.google\.Chrome\)/);
+ await window.computer.getApp("com.google.Chrome");
+ assert.doesNotMatch(window.__drain().text, /App usage notes/);
+ await window.computer.getApp("notion.id");
+ assert.match(window.__drain().text, /App usage notes \(notion\.id\)/);
+});
+
+test("silent observations do not consume an app's one-time notes", async () => {
+ const window = runtime();
+ const app = await window.computer.getApp("com.tinyspeck.slackmacgap", { emit: false });
+ assert.equal(window.__drain().text, "");
+ await app.getAXState();
+ assert.match(window.__drain().text, /App usage notes/);
+ await app.getAXState();
+ assert.doesNotMatch(window.__drain().text, /App usage notes/);
+});
+
+test("unknown apps have no invented notes, and a new runtime emits notes again", async () => {
+ const window = runtime();
+ await window.computer.getApp("test.unknown");
+ assert.doesNotMatch(window.__drain().text, /App usage notes/);
+ await window.computer.getApp("com.apple.finder");
+ assert.match(window.__drain().text, /App usage notes/);
+ const next = runtime();
+ await next.computer.getApp("com.apple.finder");
+ assert.match(next.__drain().text, /App usage notes/);
+});
+
+test("setValue forwards explicit submission and rejects invalid options before sending", async () => {
+ const calls = [];
+ const window = runtime((command) => calls.push(JSON.parse(JSON.stringify(command))));
+ const app = await window.computer.getApp("test.app", { emit: false });
+ await app.setValue(7, 42, { submit: false });
+ assert.deepEqual(calls.at(-1).actions, [{ type: "set_value", index: 7, value: "42", submit: false }]);
+ await app.setValue(8, "search", { submit: true });
+ assert.equal(calls.at(-1).actions[0].submit, true);
+ await app.setValue(9, "unchosen");
+ assert.equal(Object.hasOwn(calls.at(-1).actions[0], "submit"), false);
+ const before = calls.length;
+ assert.throws(() => app.setValue(7, "bad", { submit: "true" }), /must be true or false/);
+ assert.equal(calls.length, before);
+});
+
+test("documentation can be recovered without resetting existing app objects", async () => {
+ const calls = [];
+ const window = runtime((command) => calls.push(JSON.parse(JSON.stringify(command))));
+ const app = await window.computer.getApp("test.app", { emit: false });
+ const before = calls.length;
+ window.computer.docs();
+ assert.equal(window.__drain().text, window.__docs);
+ assert.equal(calls.length, before);
+ await app.setValue(7, "after documentation", { submit: false });
+ assert.equal(calls.at(-1).app, "test.app");
+ assert.equal(calls.at(-1).actions[0].value, "after documentation");
+});
diff --git a/desktop/test/runtime.electron.mjs b/desktop/test/runtime.electron.mjs
index eee573ef0..414c5549d 100644
--- a/desktop/test/runtime.electron.mjs
+++ b/desktop/test/runtime.electron.mjs
@@ -1,12 +1,13 @@
import assert from "node:assert/strict";
import { app, nativeImage } from "electron";
-import { mkdtemp, writeFile, chmod, rm } from "node:fs/promises";
+import { mkdtemp, writeFile, readFile, chmod, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
(async () => {
const directory = await mkdtemp(path.join(tmpdir(), "qm-runtime-test-"));
const binary = path.join(directory, "desk.mjs");
+ const acts = path.join(directory, "acts.jsonl");
process.env.QM_DESK_BIN = binary;
process.env.QM_DESK_CURSOR = "0";
const screenshot = nativeImage
@@ -17,9 +18,11 @@ import path from "node:path";
binary,
`#!/usr/bin/env node
import { createInterface } from 'node:readline';
+import { appendFileSync } from 'node:fs';
createInterface({input: process.stdin}).on('line', line => {
const {id,cmd} = JSON.parse(line);
- const observation = {app:'Fixture',window:'Fixture window',frame:{w:100,h:100},elements:1,tree:'[1] Button',diffed:false,removed:[],...(cmd.screenshot ? {screenshot:{png_base64:${JSON.stringify(screenshot)}}}: {})};
+ if (cmd.type === 'act') appendFileSync(${JSON.stringify(acts)}, JSON.stringify(cmd.actions)+'\\n');
+ const observation = {app:'Fixture',pid:1,window_id:2,window:'Fixture window',frame:{x:10,y:20,w:100,h:100},elements:1,tree:'[1] Button',diffed:false,removed:[],...(cmd.diff ? {tree:'~ [1] Button',diffed:true,removed:[3,4,5,9]} : {}),...(cmd.screenshot ? {screenshot:{png_base64:${JSON.stringify(screenshot)},frame:{x:0,y:20,w:200,h:100}}}: {})};
const reply = cmd.type === 'apps' ? {type:'apps',apps:[{name:'Fixture',bundle_id:'test.fixture',pid:1,active:true}]} : cmd.type === 'observe' ? {type:'observation',...observation} : {type:'acted',completed:1, failure:null};
process.stdout.write(JSON.stringify({id,reply})+'\\n');
});
@@ -34,6 +37,7 @@ createInterface({input: process.stdin}).on('line', line => {
try {
let result = await call("var answer = 41; answer + 1");
assert.match(result.content, /computer API/);
+ assert.match(result.content, /app\.paste\(text/);
assert.match(result.content, /=> 42/);
result = await call("answer + 2");
assert.match(result.content, /=> 43/);
@@ -43,7 +47,21 @@ createInterface({input: process.stdin}).on('line', line => {
"var fixture = await computer.getApp('Fixture'); await fixture.click(1); await fixture.getAXState()",
);
assert.match(result.content, /Fixture window/);
- assert.equal(result.content.split("Fixture window").length - 1, 1);
+ assert.equal(result.content.split("Fixture window").length - 1, 2);
+ assert.match(result.content, /changes since last state \(\+ added, ~ changed\)/);
+ assert.match(result.content, /removed: 3-5, 9/);
+ await call(
+ "await fixture.typeText('a\\tb\\n'); await fixture.paste('# Hi\\n\\n- one', {format: 'markdown'}); await fixture.paste(42)",
+ );
+ const sent = (await readFile(acts, "utf8"))
+ .trim()
+ .split("\n")
+ .map((l) => JSON.parse(l));
+ assert.deepEqual(sent.slice(-3), [
+ [{ type: "type_text", text: "a\tb\n" }],
+ [{ type: "paste", text: "# Hi\n\n- one", format: "markdown" }],
+ [{ type: "paste", text: "42", format: "text" }],
+ ]);
result = await call("await fixture.getScreenshot()");
assert.equal(result.images?.[0]?.mimeType, "image/jpeg");
assert.ok(result.images[0].data.length > 0);
@@ -51,6 +69,13 @@ createInterface({input: process.stdin}).on('line', line => {
result = await call("await fixture.getAXStateAndScreenshot({disableDiffing:true}); void 0");
assert.match(result.content, /Fixture window/);
assert.equal(result.images.length, 1);
+ await call("await fixture.click([1, 0])");
+ const clicked = (await readFile(acts, "utf8"))
+ .trim()
+ .split("\n")
+ .map((l) => JSON.parse(l))
+ .at(-1)[0].target.point;
+ assert.deepEqual([Math.round(clicked.x), Math.round(clicked.y)], [190, 0]);
result = await call("for (let i=0;i<5;i++) await fixture.getScreenshot()");
assert.equal(result.images.length, 4);
assert.match(result.content, /screenshot dropped/);
@@ -74,7 +99,7 @@ createInterface({input: process.stdin}).on('line', line => {
result = await call("6 * 7", "fourth");
assert.match(result.content, /=> 42/);
console.log(
- "PASS: persistent JavaScript, native bridge, screenshots, reset, exclusive turns, cancellation, timeout recovery",
+ "PASS: persistent JavaScript, native bridge, typing and paste requests, screenshots, reset, exclusive turns, cancellation, timeout recovery",
);
} finally {
stopComputer();
diff --git a/plugins/chassis/src/desktop-computer.ts b/plugins/chassis/src/desktop-computer.ts
index ee0c04838..53a9c8d7b 100644
--- a/plugins/chassis/src/desktop-computer.ts
+++ b/plugins/chassis/src/desktop-computer.ts
@@ -1,7 +1,7 @@
export const desktopComputerTool = {
name: "desktop",
description:
- "Run JavaScript to observe and control apps on the user's Mac through the computer API. Top-level await is supported; variables persist between calls within this turn. The first call returns API documentation. Start with await computer.listApps() or await computer.getApp(name). Console output, observations, and screenshots are returned. Set reset to clear JavaScript state. Only human-initiated desktop turns with a live connection have access.",
+ "Run JavaScript to observe and control apps on the user's Mac through the computer API. Prefer an available connector or API when it can complete the task. Top-level await is supported; variables persist between calls within this turn. The first call returns API documentation. Start with await computer.listApps() or await computer.getApp(name). Console output, observations, and screenshots are returned. Set reset to clear JavaScript state. Only human-initiated desktop turns with a live connection have access.",
timeoutMs: 60_000,
inputSchema: {
type: "object",