Skip to content

Run a virtual iPhone on an Apple Silicon Mac.

Swift

Scroll and hover natively in the VM window on iPad guests

Jackyea753f496efc
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift
index 39ec6ec9..cc1d0a58 100644
--- a/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift
+++ b/VPhoneExecutable/VPhoneVirtualization/UI/GuestCommunication/VPhoneGuestControl.swift
@@ -189,6 +189,8 @@ final class VPhoneGuestControl {
monitor = nil
orderedInput?.cancel()
orderedInput = nil
+ openScroll = nil
+ pendingPointer = nil
orderedLocation?.cancel()
orderedLocation = nil
device = nil
@@ -340,6 +342,80 @@ final class VPhoneGuestControl {
])
}
+ /// Whether the guest scrolls under a virtual pointer (`input.pointer`,
+ /// `input.scroll`), as an iPad with a trackpad does.
+ var supportsPointerScroll: Bool {
+ isConnected && guestCapabilities.contains("pointer_scroll")
+ }
+
+ @ObservationIgnored private var pointerInFlight = false
+ @ObservationIgnored private var pendingPointer: CGPoint?
+
+ /// Puts the guest's pointer at `point`, 0...1 of the interface as shown.
+ /// Not in the ordered input queue: where the pointer sits does not depend
+ /// on the order of touches, and a mouse that keeps moving would hold every
+ /// click behind it. One request at a time; a newer position replaces the
+ /// one waiting.
+ func movePointer(to point: CGPoint) {
+ guard !pointerInFlight else {
+ pendingPointer = point
+ return
+ }
+ pointerInFlight = true
+ Task {
+ _ = try? await call("input.pointer", params: ["x": Double(point.x), "y": Double(point.y)])
+ pointerInFlight = false
+ if let next = pendingPointer {
+ pendingPointer = nil
+ movePointer(to: next)
+ }
+ }
+ }
+
+ /// The `changed` deltas waiting in the input queue, added up until that
+ /// request goes out.
+ private final class PendingScroll {
+ var delta = CGVector.zero
+ }
+
+ @ObservationIgnored private var openScroll: PendingScroll?
+
+ /// One native scroll event, in order with the other input. `delta` is in
+ /// interface points with a browser wheel event's signs. `changed` deltas
+ /// that arrive while one waits in the queue are added to it, so a guest
+ /// slower than the trackpad gets fewer, larger deltas and loses none.
+ func sendScroll(phase: String, delta: CGVector = .zero, pointer: CGPoint? = nil) {
+ if phase == "changed" {
+ if let open = openScroll {
+ open.delta.dx += delta.dx
+ open.delta.dy += delta.dy
+ return
+ }
+ let open = PendingScroll()
+ open.delta = delta
+ openScroll = open
+ let previous = orderedInput
+ orderedInput = Task {
+ await previous?.value
+ if openScroll === open {
+ openScroll = nil
+ }
+ guard !Task.isCancelled else { return }
+ let params: [String: Any] = ["phase": "changed", "dx": Double(open.delta.dx), "dy": Double(open.delta.dy)]
+ do { _ = try await call("input.scroll", params: params) } catch { print("[control] input.scroll: \(error)") }
+ }
+ return
+ }
+ // A delta after this event belongs to the next scroll, not to one queued before it.
+ openScroll = nil
+ var params: [String: Any] = ["phase": phase]
+ if let pointer {
+ params["x"] = Double(pointer.x)
+ params["y"] = Double(pointer.y)
+ }
+ enqueueInput("input.scroll", params: params)
+ }
+
private func enqueueInput(_ method: String, params: [String: Any]) {
let previous = orderedInput
orderedInput = Task {
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/VPhoneVirtualMachineWindowController.swift b/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/VPhoneVirtualMachineWindowController.swift
index cbec7490..ebb4cb29 100644
--- a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/VPhoneVirtualMachineWindowController.swift
+++ b/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/VPhoneVirtualMachineWindowController.swift
@@ -48,6 +48,7 @@ class VPhoneVirtualMachineWindowController: NSObject, NSWindowDelegate {
view.capturesSystemKeys = hardwareKeyboardEnabled
view.keySender = keySender
view.control = control
+ view.screenScale = CGFloat(screenScale)
view.clipboardSync = VPhoneClipboardSync(control: control)
virtualMachineView = view
let container = VPhoneDisplayContainerView(displayView: view)
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift b/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift
index 891a2bfe..7f2e7a70 100644
--- a/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift
+++ b/VPhoneExecutable/VPhoneVirtualization/UI/VirtualMachine/VPhoneVirtualMachineView.swift
@@ -12,12 +12,16 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
var escapeIsBackGesture = true
var isTVGuest = false
var hardwareKeyboardEnabled = true
+ /// Guest pixels per interface point, to turn trackpad travel into the
+ /// guest's points for a native scroll.
+ var screenScale: CGFloat = 3
/// Whether trackpad scroll/pinch gestures are replayed as guest touches.
var trackpadGesturesEnabled = VPhoneTrackpadGestures.isEnabled {
didSet {
guard !trackpadGesturesEnabled else { return }
finishScrollTouch()
+ endNativeScroll()
endPinch()
}
}
@@ -43,6 +47,25 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
private var pinchTouchCenter: NSPoint?
private var pinchTouchRadius: CGFloat = 0
+ /// A native scroll (`input.scroll`) is under way: `began` has gone out and
+ /// `ended` not yet.
+ private var nativeScrollActive = false
+ /// `ended` waiting to see whether AppKit's momentum follows the lift.
+ private var nativeScrollEndWork: DispatchWorkItem?
+ /// A click ended the scroll; the rest of its momentum is dropped until the
+ /// fingers start a new one.
+ private var nativeScrollMomentumDropped = false
+ private var pointerTrackingArea: NSTrackingArea?
+
+ /// How long after the fingers lift a scroll waits for AppKit's momentum
+ /// before it ends. Momentum starts on the next frame when it starts at all.
+ private static let nativeScrollMomentumWait: TimeInterval = 0.05
+ /// How long a scroll from a wheel without gesture phases stays open after
+ /// its last event, as the web console does.
+ private static let nativeScrollIdleEnd: TimeInterval = 0.12
+ /// Points per line of a wheel that scrolls in lines (no precise deltas).
+ private static let nativeScrollLine: CGFloat = 16
+
/// Guest points travelled per point of trackpad delta. Above 1 because a
/// single trackpad swipe produces far less travel than a finger covers on a
/// phone screen: at 1:1 an ordinary swipe could not carry a gesture that has
@@ -123,11 +146,34 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
registerForDraggedTypes([.fileURL])
}
+ override func updateTrackingAreas() {
+ super.updateTrackingAreas()
+ if let area = pointerTrackingArea {
+ removeTrackingArea(area)
+ }
+ let area = NSTrackingArea(rect: .zero, options: [.mouseMoved, .activeInKeyWindow, .inVisibleRect], owner: self)
+ addTrackingArea(area)
+ pointerTrackingArea = area
+ }
+
+ /// On a guest that scrolls natively the guest has a pointer of its own;
+ /// it follows the mouse, so the guest sees hover as an iPad with a
+ /// trackpad does.
+ override func mouseMoved(with event: NSEvent) {
+ super.mouseMoved(with: event)
+ guard trackpadGesturesEnabled, let control, control.supportsPointerScroll else { return }
+ control.movePointer(to: interfacePoint(convert(event.locationInWindow, from: nil)))
+ }
+
override func mouseDown(with event: NSEvent) {
// Clicking the VM display should always restore keyboard focus.
window?.makeFirstResponder(self)
// A synthetic trackpad finger would fight this touch for the same slot.
finishScrollTouch()
+ if nativeScrollActive {
+ endNativeScroll()
+ nativeScrollMomentumDropped = true
+ }
releaseSecondaryPress()
let localPoint = convert(event.locationInWindow, from: nil)
currentTouchSwipeAim = hitTestEdge(at: localPoint)
@@ -219,6 +265,10 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
/// Replay a trackpad scroll as a guest finger that presses under the pointer
/// and follows the physical direction of the gesture.
override func scrollWheel(with event: NSEvent) {
+ if trackpadGesturesEnabled, let control, control.supportsPointerScroll {
+ scrollNatively(event, control: control)
+ return
+ }
guard trackpadGesturesEnabled, event.hasPreciseScrollingDeltas else {
super.scrollWheel(with: event)
return
@@ -266,6 +316,85 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
}
}
+ /// Scroll the guest the way an iPad with a trackpad scrolls: the view under
+ /// its pointer moves by the gesture's travel, and AppKit's momentum is sent
+ /// as more travel, since the guest does not decelerate after `ended`. So a
+ /// swipe runs on as far as it would in a Mac app.
+ private func scrollNatively(_ event: NSEvent, control: VPhoneGuestControl) {
+ nativeScrollEndWork?.cancel()
+ nativeScrollEndWork = nil
+ let momentum = !event.momentumPhase.isEmpty
+ if momentum, nativeScrollMomentumDropped {
+ return
+ }
+ if event.phase == .began {
+ // A new gesture: whatever was still running on is over.
+ nativeScrollMomentumDropped = false
+ endNativeScroll()
+ }
+ let delta = nativeScrollDelta(of: event)
+ if !nativeScrollActive, delta != .zero {
+ nativeScrollActive = true
+ control.sendScroll(phase: "began", pointer: interfacePoint(convert(event.locationInWindow, from: nil)))
+ }
+ if nativeScrollActive, delta != .zero {
+ control.sendScroll(phase: "changed", delta: delta)
+ }
+ if momentum {
+ if event.momentumPhase.contains(.ended) || event.momentumPhase.contains(.cancelled) {
+ endNativeScroll()
+ }
+ } else if event.phase.contains(.ended) || event.phase.contains(.cancelled) {
+ scheduleNativeScrollEnd(after: Self.nativeScrollMomentumWait)
+ } else if event.phase.isEmpty {
+ // A wheel without gesture phases never says when it stops.
+ scheduleNativeScrollEnd(after: Self.nativeScrollIdleEnd)
+ }
+ }
+
+ private func scheduleNativeScrollEnd(after delay: TimeInterval) {
+ guard nativeScrollActive else { return }
+ let work = DispatchWorkItem { [weak self] in
+ self?.endNativeScroll()
+ }
+ nativeScrollEndWork = work
+ DispatchQueue.main.asyncAfter(deadline: .now() + delay, execute: work)
+ }
+
+ private func endNativeScroll() {
+ nativeScrollEndWork?.cancel()
+ nativeScrollEndWork = nil
+ guard nativeScrollActive else { return }
+ nativeScrollActive = false
+ control?.sendScroll(phase: "ended")
+ }
+
+ /// The event's travel in guest interface points, with a browser wheel
+ /// event's signs (positive y toward the end of the content), which is the
+ /// opposite of AppKit's. Natural scrolling is already in both. The travel
+ /// is the window's, and the window shows the interface upright, so it
+ /// needs no turning; one window point is as many guest points as the drawn
+ /// display is smaller than the guest's.
+ private func nativeScrollDelta(of event: NSEvent) -> CGVector {
+ let unit = event.hasPreciseScrollingDeltas ? 1 : Self.nativeScrollLine
+ let geometry = displayGeometry
+ let pixels = recordingGraphicsDisplay?.sizeInPixels ?? .zero
+ var scale: CGFloat = 1
+ if pixels.width > 0, geometry.displayRect.width > 0, screenScale > 0 {
+ scale = pixels.width / screenScale / geometry.displayRect.width
+ }
+ return CGVector(dx: -event.scrollingDeltaX * unit * scale, dy: -event.scrollingDeltaY * unit * scale)
+ }
+
+ /// A view point as `input.pointer` takes it: 0...1 of the interface as the
+ /// window shows it, from the top-left corner.
+ private func interfacePoint(_ localPoint: NSPoint) -> CGPoint {
+ let panel = normalizeCoordinate(localPoint)
+ let unit = CGSize(width: 1, height: 1)
+ let screen = screenOrientation.screenPoint(fromPanel: CGPoint(x: panel.x, y: 1 - panel.y), panel: unit)
+ return CGPoint(x: min(max(screen.x, 0), 1), y: min(max(1 - screen.y, 0), 1))
+ }
+
/// Replay a trackpad pinch as two guest fingers spreading or closing around
/// the pointer position.
override func magnify(with event: NSEvent) {
@@ -273,6 +402,8 @@ class VPhoneVirtualMachineView: VZVirtualMachineView {
super.magnify(with: event)
return
}
+ // A native scroll and a pinch never overlap, like a synthetic drag and one.
+ endNativeScroll()
// A synthetic drag owns the touch surface; ignoring the pinch keeps the
// guest from seeing one- and two-finger touches interleaved.
guard scrollTouchPoint == nil else { return }