Skip to content

Run a virtual iPhone on an Apple Silicon Mac.

Swift

Track the mouse without a round trip and hide the Mac's cursor over the display

Jackyfa0eb5032ca5
diff --git a/Research/vphoned_http_api.md b/Research/vphoned_http_api.md
index 1b75ae44..4e5894a5 100644
--- a/Research/vphoned_http_api.md
+++ b/Research/vphoned_http_api.md
@@ -324,7 +324,7 @@ request carries `"force": true`.
| --- | --- |
| Device | `device.snapshot`, `device.info` (snapshot plus network, screen, rotation, brightness, volume, low power, Developer Mode, agent, `device_name` `{name, own_name}`: the pinned name and the guest's own ComputerName), `device.screen`, `device.network`, `device.ioreg {plane}`, `device.environment`, `device.basebin {archive?}` |
| Display, audio | `display.brightness {value?}`, `display.rotation {orientation?}`, `display.orientation` (`{degrees, source}`: SpringBoard's interface orientation without a screen capture, for polling; `degrees` is null while SpringBoard has no answer; capability `display_orientation`), `display.rotation_lock {locked}`, `display.auto_lock` (`{auto_lock_seconds, never, lock_screen_minimum_seconds}`: Auto-Lock and the Lock Screen timeout vphoned keeps in step with it, see below; capability `display_auto_lock`), `screen.unlock {passcode?, timeout?}` (`{locked, screen_off, was_locked, was_screen_off}`: the display on and the Lock Screen passed, see below; capability `screen_unlock`), `audio.volume {value?, category?}`, `audio.state` |
-| Input | `input.touch`, `input.hid`, `input.button {name}`, `input.key {name}`, `input.type {text, delay_ms?}`, `input.paste {text}`, `input.tap`, `input.double_tap`, `input.long_press`, `input.swipe`, `input.drag {points}`, `input.touch_sequence {events}` — gesture coordinates are screen points; `input.pointer {x, y}`, `input.scroll {phase, dx?, dy?, x?, y?}` (capability `pointer_scroll`, see below) |
+| Input | `input.touch`, `input.hid`, `input.button {name}`, `input.key {name}`, `input.type {text, delay_ms?}`, `input.paste {text}`, `input.tap`, `input.double_tap`, `input.long_press`, `input.swipe`, `input.drag {points}`, `input.touch_sequence {events}` — gesture coordinates are screen points; `input.pointer {x, y, seq?}`, `input.scroll {phase, dx?, dy?, x?, y?}` (capability `pointer_scroll`, see below) |
| UI | `ui.tree` (alias `accessibility.tree`), `ui.element_at`, `ui.tap_element`, `ui.wait`, `ui.wait_gone`, `ui.ocr {languages?, min_confidence?}`, `ui.describe`, `screen.screenshot` |
| Processes | `processes.list {filter?}`, `processes.kill {pid, signal?}` **force**, `memory.jetsam`, `memory.pressure` (only the three kernel memory sysctls, for polling) |
| launchd | `services.list`, `status`, `print`, `dump`, `disabled`, `start`, `enable`, `load`, `profile` (see below); `services.stop`, `disable`, `remove`, `signal`, `unload`, `profile.apply {profile, groups?, allow?}` **force**; `launchd.getenv`, `setenv`, `unsetenv`. On tvOS, which never enables user sessions, they read the system domain alone; before icli 0.8.6 the foreground-user lookup trapped there and killed vphoned |
@@ -423,8 +423,12 @@ vphoned creates a virtual pointing device (a HID user device with an absolute
position, `VPhoneDaemon/Native/vphoned_pointer.m`); iPadOS then shows a pointer,
which hides itself after a few seconds without movement.
-- `input.pointer {x, y}` puts the pointer at x, y: 0...1 from the top-left
- corner of the interface as it is shown, in any orientation.
+- `input.pointer {x, y, seq?}` puts the pointer at x, y: 0...1 from the
+ top-left corner of the interface as it is shown, in any orientation.
+ Requests run concurrently, so a client that sends moves without waiting
+ for each answer numbers them with an increasing `seq` (a whole number up
+ to 2^53); a move that lands after a higher one is dropped, and still
+ answers. Without `seq` every move is applied.
- `input.scroll {phase, dx?, dy?, x?, y?}` sends one scroll event to the view
under the pointer, after moving the pointer to x, y when they are given.
`phase` is `began`, `changed`, `ended` or `cancelled`: `began` first, a
diff --git a/Skills/vphone-guest-control/references/rpc-methods.md b/Skills/vphone-guest-control/references/rpc-methods.md
index 3d9ac3c5..65516125 100644
--- a/Skills/vphone-guest-control/references/rpc-methods.md
+++ b/Skills/vphone-guest-control/references/rpc-methods.md
@@ -73,7 +73,7 @@ invalid-request error, read the source named there instead of guessing keys.
| `input.key` | `name` (`return`, `cmd+v`, …) |
| `input.type` | `text`, `delay_ms?` (30) |
| `input.paste` | `text` |
-| `input.pointer` | `x`, `y` (0...1 of the interface as shown; iPad, capability `pointer_scroll`) |
+| `input.pointer` | `x`, `y` (0...1 of the interface as shown; iPad, capability `pointer_scroll`), `seq?` (increasing; a late move is dropped) |
| `input.scroll` | `phase` (`began`/`changed`/`ended`/`cancelled`), `dx?`, `dy?` (points, wheel signs), `x?`, `y?` |
## UI inspection
diff --git a/VPhoneDaemon/Daemon/GuestAPI.swift b/VPhoneDaemon/Daemon/GuestAPI.swift
index bf3edfb9..c7afc0db 100644
--- a/VPhoneDaemon/Daemon/GuestAPI.swift
+++ b/VPhoneDaemon/Daemon/GuestAPI.swift
@@ -336,12 +336,14 @@ enum GuestAPI {
return ["phase": phase.rawValue, "fingers": 2]
case "input.pointer":
let point: GuestPointerScroll.Point
+ let sequence: UInt64
do {
point = try GuestPointerScroll.point(params)
+ sequence = try GuestPointerScroll.sequence(params)
} catch {
throw GuestAPIError.invalidRequest(error.message)
}
- try pointerResult(vp_pointer_move(point.x, point.y))
+ try pointerResult(vp_pointer_move(point.x, point.y, sequence))
return ["x": point.x, "y": point.y]
case "input.scroll":
let scroll: GuestPointerScroll.Scroll
@@ -351,7 +353,7 @@ enum GuestAPI {
throw GuestAPIError.invalidRequest(error.message)
}
if let point = scroll.pointer {
- try pointerResult(vp_pointer_move(point.x, point.y))
+ try pointerResult(vp_pointer_move(point.x, point.y, 0))
}
try pointerResult(vp_pointer_scroll(scroll.phase.bits, scroll.dx, scroll.dy))
return ["phase": scroll.phase.rawValue]
diff --git a/VPhoneDaemon/Daemon/GuestPointerScroll.swift b/VPhoneDaemon/Daemon/GuestPointerScroll.swift
index 1fe5847b..c580c368 100644
--- a/VPhoneDaemon/Daemon/GuestPointerScroll.swift
+++ b/VPhoneDaemon/Daemon/GuestPointerScroll.swift
@@ -63,6 +63,22 @@ enum GuestPointerScroll {
return Point(x: x, y: y)
}
+ /// The largest `seq`: what a JSON number holds exactly.
+ static let maximumSequence: UInt64 = 1 << 53
+
+ /// `input.pointer`'s optional `seq`, 0 when there is none. A client that
+ /// sends moves without waiting for each answer numbers them in increasing
+ /// order; one that arrives after a higher number is dropped.
+ static func sequence(_ params: [String: Any]) throws(Problem) -> UInt64 {
+ guard let value = params["seq"] else { return 0 }
+ guard let sequence = number(value), sequence >= 0, sequence <= Double(maximumSequence),
+ sequence.rounded() == sequence
+ else {
+ throw Problem(message: "seq must be a whole number within 0...\(maximumSequence)")
+ }
+ return UInt64(sequence)
+ }
+
static func scroll(_ params: [String: Any]) throws(Problem) -> Scroll {
guard let name = params["phase"] as? String, let phase = Phase(rawValue: name) else {
throw Problem(message: "phase must be began, changed, ended or cancelled")
diff --git a/VPhoneDaemon/Native/Include/VphonedNative.h b/VPhoneDaemon/Native/Include/VphonedNative.h
index ddf96c22..b07db652 100644
--- a/VPhoneDaemon/Native/Include/VphonedNative.h
+++ b/VPhoneDaemon/Native/Include/VphonedNative.h
@@ -41,8 +41,10 @@ enum {
};
/// Put the virtual pointer at x, y: 0...1 from the top-left corner of the
-/// interface as it is shown. Creates the pointing device on first use.
-int vp_pointer_move(double x, double y);
+/// interface as it is shown. Creates the pointing device on first use. A
+/// nonzero `sequence` not above the last one applied is a move that arrived
+/// late: it is dropped and the call still succeeds. 0 always moves.
+int vp_pointer_move(double x, double y, uint64_t sequence);
/// One scroll event at the pointer. `phase` is an IOHIDEventPhaseBits value;
/// dx and dy are interface points, with a browser wheel event's signs.
diff --git a/VPhoneDaemon/Native/vphoned_pointer.m b/VPhoneDaemon/Native/vphoned_pointer.m
index c6f7a0fb..55711e14 100644
--- a/VPhoneDaemon/Native/vphoned_pointer.m
+++ b/VPhoneDaemon/Native/vphoned_pointer.m
@@ -52,6 +52,7 @@ static bool gPointerLoaded;
static IOHIDEventSystemClientRef gPointerClient;
static IOHIDUserDeviceRef gPointerDevice;
static uint64_t gPointerSender;
+static uint64_t gPointerLastSequence;
static NSString *const kPointerProduct = @"vphone pointer";
@@ -155,11 +156,17 @@ static dispatch_queue_t pointer_queue(void) {
return gPointerQueue;
}
-int vp_pointer_move(double x, double y) {
+int vp_pointer_move(double x, double y, uint64_t sequence) {
__block int result;
dispatch_sync(pointer_queue(), ^{
result = ensure_device();
if (result != VP_POINTER_OK) return;
+ // Requests run concurrently, so a move sent earlier can land later;
+ // putting the pointer back where it was would make it jump.
+ if (sequence) {
+ if (sequence <= gPointerLastSequence) return;
+ gPointerLastSequence = sequence;
+ }
uint16_t px = (uint16_t)lround(fmin(fmax(x, 0), 1) * kPointerMaximum);
uint16_t py = (uint16_t)lround(fmin(fmax(y, 0), 1) * kPointerMaximum);
const uint8_t report[7] = {0, px & 0xFF, px >> 8, py & 0xFF, py >> 8, 0, 0};
diff --git a/VPhoneDaemon/Tests/LogicTests.swift b/VPhoneDaemon/Tests/LogicTests.swift
index 00fd0e06..83678445 100644
--- a/VPhoneDaemon/Tests/LogicTests.swift
+++ b/VPhoneDaemon/Tests/LogicTests.swift
@@ -384,5 +384,11 @@ enum LogicTests {
expectEqual(scrollProblem(["phase": "began", "x": -0.1, "y": 0.5]), point)
expectEqual(try? GuestPointerScroll.point(["x": 0.25, "y": 0.75]), GuestPointerScroll.Point(x: 0.25, y: 0.75))
expect((try? GuestPointerScroll.point(["x": false, "y": 0])) == nil, "a boolean is not a coordinate")
+ expectEqual(try? GuestPointerScroll.sequence([:]), 0, "no seq always moves")
+ expectEqual(try? GuestPointerScroll.sequence(["seq": 42]), 42)
+ expectEqual(try? GuestPointerScroll.sequence(["seq": 9_007_199_254_740_992.0]), 1 << 53)
+ for bad: Any in [-1, 1.5, true, "7", 9_007_199_254_740_994.0] {
+ expect((try? GuestPointerScroll.sequence(["seq": bad])) == nil, "seq \(bad) is refused")
+ }
}
}
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift
index cc1d0a58..22421db1 100644
--- a/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift
+++ b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift
@@ -348,23 +348,34 @@ final class VPhoneGuestControl {
isConnected && guestCapabilities.contains("pointer_scroll")
}
- @ObservationIgnored private var pointerInFlight = false
+ @ObservationIgnored private var pointersInFlight = 0
@ObservationIgnored private var pendingPointer: CGPoint?
+ @ObservationIgnored private var lastPointerSequence: UInt64 = 0
+
+ /// Moves in flight at once. Each request is its own connection; waiting
+ /// for one answer before the next put a round trip behind every move.
+ private static let maximumPointersInFlight = 4
/// Puts the guest's pointer at `point`, 0...1 of the interface as shown.
/// Not in the ordered input queue: where the pointer sits does not depend
/// on the order of touches, and a mouse that keeps moving would hold every
- /// click behind it. One request at a time; a newer position replaces the
- /// one waiting.
+ /// click behind it. Moves go out at once, numbered so that vphoned drops
+ /// one that lands after a later one; past `maximumPointersInFlight` the
+ /// newest position waits for the next free slot.
func movePointer(to point: CGPoint) {
- guard !pointerInFlight else {
+ guard pointersInFlight < Self.maximumPointersInFlight else {
pendingPointer = point
return
}
- pointerInFlight = true
+ // Microseconds of uptime: increasing across reconnects and restarts of
+ // this process, which a guest that keeps running may still remember.
+ let now = UInt64(ProcessInfo.processInfo.systemUptime * 1_000_000)
+ lastPointerSequence = max(now, lastPointerSequence + 1)
+ let params: [String: Any] = ["x": Double(point.x), "y": Double(point.y), "seq": lastPointerSequence]
+ pointersInFlight += 1
Task {
- _ = try? await call("input.pointer", params: ["x": Double(point.x), "y": Double(point.y)])
- pointerInFlight = false
+ _ = try? await call("input.pointer", params: params)
+ pointersInFlight -= 1
if let next = pendingPointer {
pendingPointer = nil
movePointer(to: next)
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift b/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift
index 7f2e7a70..4bff9dc5 100644
--- a/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift
+++ b/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift
@@ -151,7 +151,9 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
if let area = pointerTrackingArea {
removeTrackingArea(area)
}
- let area = NSTrackingArea(rect: .zero, options: [.mouseMoved, .activeInKeyWindow, .inVisibleRect], owner: self)
+ let area = NSTrackingArea(
+ rect: .zero, options: [.mouseMoved, .cursorUpdate, .activeInKeyWindow, .inVisibleRect], owner: self,
+ )
addTrackingArea(area)
pointerTrackingArea = area
}
@@ -161,10 +163,29 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
/// trackpad does.
override func mouseMoved(with event: NSEvent) {
super.mouseMoved(with: event)
- guard trackpadGesturesEnabled, let control, control.supportsPointerScroll else { return }
+ guard guestPointerFollowsMouse, let control else { return }
+ Self.hiddenCursor.set()
control.movePointer(to: interfacePoint(convert(event.locationInWindow, from: nil)))
}
+ /// The guest's pointer stands in for the Mac's over the display, so the
+ /// Mac's is hidden there, as Universal Control hides it on the iPad.
+ override func cursorUpdate(with event: NSEvent) {
+ if guestPointerFollowsMouse {
+ Self.hiddenCursor.set()
+ } else {
+ super.cursorUpdate(with: event)
+ }
+ }
+
+ private var guestPointerFollowsMouse: Bool {
+ trackpadGesturesEnabled && control?.supportsPointerScroll == true
+ }
+
+ /// A cursor with nothing to draw. `NSCursor.hide()` is counted and
+ /// app-wide, so it would have to be balanced on every way out of the view.
+ private static let hiddenCursor = NSCursor(image: NSImage(size: NSSize(width: 1, height: 1)), hotSpot: .zero)
+
override func mouseDown(with event: NSEvent) {
// Clicking the VM display should always restore keyboard focus.
window?.makeFirstResponder(self)
@@ -174,6 +195,11 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
endNativeScroll()
nativeScrollMomentumDropped = true
}
+ // A click is a touch, and iPadOS hides its pointer under a finger: the
+ // Mac's shows where the drag is until the mouse moves freely again.
+ if guestPointerFollowsMouse {
+ NSCursor.arrow.set()
+ }
releaseSecondaryPress()
let localPoint = convert(event.locationInWindow, from: nil)
currentTouchSwipeAim = hitTestEdge(at: localPoint)
@@ -197,6 +223,11 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
super.mouseUp(with: event)
}
currentTouchSwipeAim = 0
+ // Bring the guest's pointer back where the finger lifted.
+ if guestPointerFollowsMouse, let control {
+ Self.hiddenCursor.set()
+ control.movePointer(to: interfacePoint(localPoint))
+ }
}
// MARK: - Secondary Click