Run a virtual iPhone on an Apple Silicon Mac.
Swift
Track the mouse without a round trip and hide the Mac's cursor over the display
Jackyfa0eb5032ca5
diff --git a/Research/vphoned_http_api.md b/Research/vphoned_http_api.mdindex 1b75ae44..4e5894a5 100644--- a/Research/vphoned_http_api.md+++ b/Research/vphoned_http_api.md@@ -324,7 +324,7 @@ request carries `"force": true`.| --- | --- || Device | `device.snapshot`, `device.info` (snapshot plus network, screen, rotation, brightness, volume, low power, Developer Mode, agent, `device_name` `{name, own_name}`: the pinned name and the guest's own ComputerName), `device.screen`, `device.network`, `device.ioreg {plane}`, `device.environment`, `device.basebin {archive?}` || Display, audio | `display.brightness {value?}`, `display.rotation {orientation?}`, `display.orientation` (`{degrees, source}`: SpringBoard's interface orientation without a screen capture, for polling; `degrees` is null while SpringBoard has no answer; capability `display_orientation`), `display.rotation_lock {locked}`, `display.auto_lock` (`{auto_lock_seconds, never, lock_screen_minimum_seconds}`: Auto-Lock and the Lock Screen timeout vphoned keeps in step with it, see below; capability `display_auto_lock`), `screen.unlock {passcode?, timeout?}` (`{locked, screen_off, was_locked, was_screen_off}`: the display on and the Lock Screen passed, see below; capability `screen_unlock`), `audio.volume {value?, category?}`, `audio.state` |-| Input | `input.touch`, `input.hid`, `input.button {name}`, `input.key {name}`, `input.type {text, delay_ms?}`, `input.paste {text}`, `input.tap`, `input.double_tap`, `input.long_press`, `input.swipe`, `input.drag {points}`, `input.touch_sequence {events}` — gesture coordinates are screen points; `input.pointer {x, y}`, `input.scroll {phase, dx?, dy?, x?, y?}` (capability `pointer_scroll`, see below) |+| Input | `input.touch`, `input.hid`, `input.button {name}`, `input.key {name}`, `input.type {text, delay_ms?}`, `input.paste {text}`, `input.tap`, `input.double_tap`, `input.long_press`, `input.swipe`, `input.drag {points}`, `input.touch_sequence {events}` — gesture coordinates are screen points; `input.pointer {x, y, seq?}`, `input.scroll {phase, dx?, dy?, x?, y?}` (capability `pointer_scroll`, see below) || UI | `ui.tree` (alias `accessibility.tree`), `ui.element_at`, `ui.tap_element`, `ui.wait`, `ui.wait_gone`, `ui.ocr {languages?, min_confidence?}`, `ui.describe`, `screen.screenshot` || Processes | `processes.list {filter?}`, `processes.kill {pid, signal?}` **force**, `memory.jetsam`, `memory.pressure` (only the three kernel memory sysctls, for polling) || launchd | `services.list`, `status`, `print`, `dump`, `disabled`, `start`, `enable`, `load`, `profile` (see below); `services.stop`, `disable`, `remove`, `signal`, `unload`, `profile.apply {profile, groups?, allow?}` **force**; `launchd.getenv`, `setenv`, `unsetenv`. On tvOS, which never enables user sessions, they read the system domain alone; before icli 0.8.6 the foreground-user lookup trapped there and killed vphoned |@@ -423,8 +423,12 @@ vphoned creates a virtual pointing device (a HID user device with an absoluteposition, `VPhoneDaemon/Native/vphoned_pointer.m`); iPadOS then shows a pointer,which hides itself after a few seconds without movement.-- `input.pointer {x, y}` puts the pointer at x, y: 0...1 from the top-left- corner of the interface as it is shown, in any orientation.+- `input.pointer {x, y, seq?}` puts the pointer at x, y: 0...1 from the+ top-left corner of the interface as it is shown, in any orientation.+ Requests run concurrently, so a client that sends moves without waiting+ for each answer numbers them with an increasing `seq` (a whole number up+ to 2^53); a move that lands after a higher one is dropped, and still+ answers. Without `seq` every move is applied.- `input.scroll {phase, dx?, dy?, x?, y?}` sends one scroll event to the viewunder the pointer, after moving the pointer to x, y when they are given.`phase` is `began`, `changed`, `ended` or `cancelled`: `began` first, adiff --git a/Skills/vphone-guest-control/references/rpc-methods.md b/Skills/vphone-guest-control/references/rpc-methods.mdindex 3d9ac3c5..65516125 100644--- a/Skills/vphone-guest-control/references/rpc-methods.md+++ b/Skills/vphone-guest-control/references/rpc-methods.md@@ -73,7 +73,7 @@ invalid-request error, read the source named there instead of guessing keys.| `input.key` | `name` (`return`, `cmd+v`, …) || `input.type` | `text`, `delay_ms?` (30) || `input.paste` | `text` |-| `input.pointer` | `x`, `y` (0...1 of the interface as shown; iPad, capability `pointer_scroll`) |+| `input.pointer` | `x`, `y` (0...1 of the interface as shown; iPad, capability `pointer_scroll`), `seq?` (increasing; a late move is dropped) || `input.scroll` | `phase` (`began`/`changed`/`ended`/`cancelled`), `dx?`, `dy?` (points, wheel signs), `x?`, `y?` |## UI inspectiondiff --git a/VPhoneDaemon/Daemon/GuestAPI.swift b/VPhoneDaemon/Daemon/GuestAPI.swiftindex bf3edfb9..c7afc0db 100644--- a/VPhoneDaemon/Daemon/GuestAPI.swift+++ b/VPhoneDaemon/Daemon/GuestAPI.swift@@ -336,12 +336,14 @@ enum GuestAPI {return ["phase": phase.rawValue, "fingers": 2]case "input.pointer":let point: GuestPointerScroll.Point+ let sequence: UInt64do {point = try GuestPointerScroll.point(params)+ sequence = try GuestPointerScroll.sequence(params)} catch {throw GuestAPIError.invalidRequest(error.message)}- try pointerResult(vp_pointer_move(point.x, point.y))+ try pointerResult(vp_pointer_move(point.x, point.y, sequence))return ["x": point.x, "y": point.y]case "input.scroll":let scroll: GuestPointerScroll.Scroll@@ -351,7 +353,7 @@ enum GuestAPI {throw GuestAPIError.invalidRequest(error.message)}if let point = scroll.pointer {- try pointerResult(vp_pointer_move(point.x, point.y))+ try pointerResult(vp_pointer_move(point.x, point.y, 0))}try pointerResult(vp_pointer_scroll(scroll.phase.bits, scroll.dx, scroll.dy))return ["phase": scroll.phase.rawValue]diff --git a/VPhoneDaemon/Daemon/GuestPointerScroll.swift b/VPhoneDaemon/Daemon/GuestPointerScroll.swiftindex 1fe5847b..c580c368 100644--- a/VPhoneDaemon/Daemon/GuestPointerScroll.swift+++ b/VPhoneDaemon/Daemon/GuestPointerScroll.swift@@ -63,6 +63,22 @@ enum GuestPointerScroll {return Point(x: x, y: y)}+ /// The largest `seq`: what a JSON number holds exactly.+ static let maximumSequence: UInt64 = 1 << 53++ /// `input.pointer`'s optional `seq`, 0 when there is none. A client that+ /// sends moves without waiting for each answer numbers them in increasing+ /// order; one that arrives after a higher number is dropped.+ static func sequence(_ params: [String: Any]) throws(Problem) -> UInt64 {+ guard let value = params["seq"] else { return 0 }+ guard let sequence = number(value), sequence >= 0, sequence <= Double(maximumSequence),+ sequence.rounded() == sequence+ else {+ throw Problem(message: "seq must be a whole number within 0...\(maximumSequence)")+ }+ return UInt64(sequence)+ }+static func scroll(_ params: [String: Any]) throws(Problem) -> Scroll {guard let name = params["phase"] as? String, let phase = Phase(rawValue: name) else {throw Problem(message: "phase must be began, changed, ended or cancelled")diff --git a/VPhoneDaemon/Native/Include/VphonedNative.h b/VPhoneDaemon/Native/Include/VphonedNative.hindex ddf96c22..b07db652 100644--- a/VPhoneDaemon/Native/Include/VphonedNative.h+++ b/VPhoneDaemon/Native/Include/VphonedNative.h@@ -41,8 +41,10 @@ enum {};/// Put the virtual pointer at x, y: 0...1 from the top-left corner of the-/// interface as it is shown. Creates the pointing device on first use.-int vp_pointer_move(double x, double y);+/// interface as it is shown. Creates the pointing device on first use. A+/// nonzero `sequence` not above the last one applied is a move that arrived+/// late: it is dropped and the call still succeeds. 0 always moves.+int vp_pointer_move(double x, double y, uint64_t sequence);/// One scroll event at the pointer. `phase` is an IOHIDEventPhaseBits value;/// dx and dy are interface points, with a browser wheel event's signs.diff --git a/VPhoneDaemon/Native/vphoned_pointer.m b/VPhoneDaemon/Native/vphoned_pointer.mindex c6f7a0fb..55711e14 100644--- a/VPhoneDaemon/Native/vphoned_pointer.m+++ b/VPhoneDaemon/Native/vphoned_pointer.m@@ -52,6 +52,7 @@ static bool gPointerLoaded;static IOHIDEventSystemClientRef gPointerClient;static IOHIDUserDeviceRef gPointerDevice;static uint64_t gPointerSender;+static uint64_t gPointerLastSequence;static NSString *const kPointerProduct = @"vphone pointer";@@ -155,11 +156,17 @@ static dispatch_queue_t pointer_queue(void) {return gPointerQueue;}-int vp_pointer_move(double x, double y) {+int vp_pointer_move(double x, double y, uint64_t sequence) {__block int result;dispatch_sync(pointer_queue(), ^{result = ensure_device();if (result != VP_POINTER_OK) return;+ // Requests run concurrently, so a move sent earlier can land later;+ // putting the pointer back where it was would make it jump.+ if (sequence) {+ if (sequence <= gPointerLastSequence) return;+ gPointerLastSequence = sequence;+ }uint16_t px = (uint16_t)lround(fmin(fmax(x, 0), 1) * kPointerMaximum);uint16_t py = (uint16_t)lround(fmin(fmax(y, 0), 1) * kPointerMaximum);const uint8_t report[7] = {0, px & 0xFF, px >> 8, py & 0xFF, py >> 8, 0, 0};diff --git a/VPhoneDaemon/Tests/LogicTests.swift b/VPhoneDaemon/Tests/LogicTests.swiftindex 00fd0e06..83678445 100644--- a/VPhoneDaemon/Tests/LogicTests.swift+++ b/VPhoneDaemon/Tests/LogicTests.swift@@ -384,5 +384,11 @@ enum LogicTests {expectEqual(scrollProblem(["phase": "began", "x": -0.1, "y": 0.5]), point)expectEqual(try? GuestPointerScroll.point(["x": 0.25, "y": 0.75]), GuestPointerScroll.Point(x: 0.25, y: 0.75))expect((try? GuestPointerScroll.point(["x": false, "y": 0])) == nil, "a boolean is not a coordinate")+ expectEqual(try? GuestPointerScroll.sequence([:]), 0, "no seq always moves")+ expectEqual(try? GuestPointerScroll.sequence(["seq": 42]), 42)+ expectEqual(try? GuestPointerScroll.sequence(["seq": 9_007_199_254_740_992.0]), 1 << 53)+ for bad: Any in [-1, 1.5, true, "7", 9_007_199_254_740_994.0] {+ expect((try? GuestPointerScroll.sequence(["seq": bad])) == nil, "seq \(bad) is refused")+ }}}diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swiftindex cc1d0a58..22421db1 100644--- a/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift+++ b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift@@ -348,23 +348,34 @@ final class VPhoneGuestControl {isConnected && guestCapabilities.contains("pointer_scroll")}- @ObservationIgnored private var pointerInFlight = false+ @ObservationIgnored private var pointersInFlight = 0@ObservationIgnored private var pendingPointer: CGPoint?+ @ObservationIgnored private var lastPointerSequence: UInt64 = 0++ /// Moves in flight at once. Each request is its own connection; waiting+ /// for one answer before the next put a round trip behind every move.+ private static let maximumPointersInFlight = 4/// Puts the guest's pointer at `point`, 0...1 of the interface as shown./// Not in the ordered input queue: where the pointer sits does not depend/// on the order of touches, and a mouse that keeps moving would hold every- /// click behind it. One request at a time; a newer position replaces the- /// one waiting.+ /// click behind it. Moves go out at once, numbered so that vphoned drops+ /// one that lands after a later one; past `maximumPointersInFlight` the+ /// newest position waits for the next free slot.func movePointer(to point: CGPoint) {- guard !pointerInFlight else {+ guard pointersInFlight < Self.maximumPointersInFlight else {pendingPointer = pointreturn}- pointerInFlight = true+ // Microseconds of uptime: increasing across reconnects and restarts of+ // this process, which a guest that keeps running may still remember.+ let now = UInt64(ProcessInfo.processInfo.systemUptime * 1_000_000)+ lastPointerSequence = max(now, lastPointerSequence + 1)+ let params: [String: Any] = ["x": Double(point.x), "y": Double(point.y), "seq": lastPointerSequence]+ pointersInFlight += 1Task {- _ = try? await call("input.pointer", params: ["x": Double(point.x), "y": Double(point.y)])- pointerInFlight = false+ _ = try? await call("input.pointer", params: params)+ pointersInFlight -= 1if let next = pendingPointer {pendingPointer = nilmovePointer(to: next)diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift b/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swiftindex 7f2e7a70..4bff9dc5 100644--- a/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift+++ b/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift@@ -151,7 +151,9 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {if let area = pointerTrackingArea {removeTrackingArea(area)}- let area = NSTrackingArea(rect: .zero, options: [.mouseMoved, .activeInKeyWindow, .inVisibleRect], owner: self)+ let area = NSTrackingArea(+ rect: .zero, options: [.mouseMoved, .cursorUpdate, .activeInKeyWindow, .inVisibleRect], owner: self,+ )addTrackingArea(area)pointerTrackingArea = area}@@ -161,10 +163,29 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {/// trackpad does.override func mouseMoved(with event: NSEvent) {super.mouseMoved(with: event)- guard trackpadGesturesEnabled, let control, control.supportsPointerScroll else { return }+ guard guestPointerFollowsMouse, let control else { return }+ Self.hiddenCursor.set()control.movePointer(to: interfacePoint(convert(event.locationInWindow, from: nil)))}+ /// The guest's pointer stands in for the Mac's over the display, so the+ /// Mac's is hidden there, as Universal Control hides it on the iPad.+ override func cursorUpdate(with event: NSEvent) {+ if guestPointerFollowsMouse {+ Self.hiddenCursor.set()+ } else {+ super.cursorUpdate(with: event)+ }+ }++ private var guestPointerFollowsMouse: Bool {+ trackpadGesturesEnabled && control?.supportsPointerScroll == true+ }++ /// A cursor with nothing to draw. `NSCursor.hide()` is counted and+ /// app-wide, so it would have to be balanced on every way out of the view.+ private static let hiddenCursor = NSCursor(image: NSImage(size: NSSize(width: 1, height: 1)), hotSpot: .zero)+override func mouseDown(with event: NSEvent) {// Clicking the VM display should always restore keyboard focus.window?.makeFirstResponder(self)@@ -174,6 +195,11 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {endNativeScroll()nativeScrollMomentumDropped = true}+ // A click is a touch, and iPadOS hides its pointer under a finger: the+ // Mac's shows where the drag is until the mouse moves freely again.+ if guestPointerFollowsMouse {+ NSCursor.arrow.set()+ }releaseSecondaryPress()let localPoint = convert(event.locationInWindow, from: nil)currentTouchSwipeAim = hitTestEdge(at: localPoint)@@ -197,6 +223,11 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {super.mouseUp(with: event)}currentTouchSwipeAim = 0+ // Bring the guest's pointer back where the finger lifted.+ if guestPointerFollowsMouse, let control {+ Self.hiddenCursor.set()+ control.movePointer(to: interfacePoint(localPoint))+ }}// MARK: - Secondary Click