Skip to content

Run a virtual iPhone on an Apple Silicon Mac.

Swift

Take icli from 0.7.3 so Setup Assistant counts as the frontmost app

Lakr59903d5bca14
diff --git a/VPhone.xcworkspace/xcshareddata/swiftpm/Package.resolved b/VPhone.xcworkspace/xcshareddata/swiftpm/Package.resolved
index ce7ac730..f5c6e55a 100644
--- a/VPhone.xcworkspace/xcshareddata/swiftpm/Package.resolved
+++ b/VPhone.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -24,8 +24,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/owngoal-dev/icli.git",
"state" : {
- "revision" : "45c92efcbee829ebbc485232122b6f94c03356da",
- "version" : "0.7.1"
+ "revision" : "ad968a099d20aa35f23b019dfbc549da674337c1",
+ "version" : "0.7.3"
}
},
{
diff --git a/VPhoneDaemon/VPhoneDaemon.xcodeproj/project.pbxproj b/VPhoneDaemon/VPhoneDaemon.xcodeproj/project.pbxproj
index 5764e37e..65419f32 100644
--- a/VPhoneDaemon/VPhoneDaemon.xcodeproj/project.pbxproj
+++ b/VPhoneDaemon/VPhoneDaemon.xcodeproj/project.pbxproj
@@ -344,8 +344,8 @@
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/owngoal-dev/icli.git";
requirement = {
- kind = exactVersion;
- version = 0.7.1;
+ kind = upToNextMajorVersion;
+ minimumVersion = 0.7.3;
};
};
C10000000000000000000072 /* XCRemoteSwiftPackageReference "swift-nio" */ = {
diff --git a/VPhoneDaemon/VPhoneDaemon.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/VPhoneDaemon/VPhoneDaemon.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
index e7d1eae2..347a70e6 100644
--- a/VPhoneDaemon/VPhoneDaemon.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
+++ b/VPhoneDaemon/VPhoneDaemon.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -6,8 +6,8 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/owngoal-dev/icli.git",
"state" : {
- "revision" : "45c92efcbee829ebbc485232122b6f94c03356da",
- "version" : "0.7.1"
+ "revision" : "ad968a099d20aa35f23b019dfbc549da674337c1",
+ "version" : "0.7.3"
}
},
{
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorDetailView.swift b/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorDetailView.swift
deleted file mode 100644
index f0b457c6..00000000
--- a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorDetailView.swift
+++ /dev/null
@@ -1,36 +0,0 @@
-import SwiftUI
-
-/// Every key of the selected record, selectable for copying.
-struct VPhoneUIInspectorDetailView: View {
- let details: [VPhoneUIInspectorDetail]
-
- var body: some View {
- if details.isEmpty {
- Text("Select an element in the table or on the screenshot to see all of its keys.")
- .font(.system(size: 11))
- .foregroundStyle(.secondary)
- .multilineTextAlignment(.center)
- .padding(12)
- .frame(maxWidth: .infinity, maxHeight: .infinity)
- } else {
- ScrollView {
- Grid(alignment: .leadingFirstTextBaseline, horizontalSpacing: 12, verticalSpacing: 2) {
- ForEach(details) { detail in
- GridRow {
- Text(verbatim: detail.key)
- .foregroundStyle(.secondary)
- .gridColumnAlignment(.trailing)
- Text(verbatim: detail.value)
- .frame(maxWidth: .infinity, alignment: .leading)
- }
- }
- }
- .font(.system(size: 11, design: .monospaced))
- .textSelection(.enabled)
- .padding(.horizontal, 12)
- .padding(.vertical, 8)
- }
- .background(Color(nsColor: .textBackgroundColor))
- }
- }
-}
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorElement.swift b/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorElement.swift
deleted file mode 100644
index 78a8ca1e..00000000
--- a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorElement.swift
+++ /dev/null
@@ -1,72 +0,0 @@
-import Foundation
-
-// MARK: - Accessibility Element
-
-/// One row of `ui.tree`. icli serializes each AX element with `label`,
-/// `identifier`, `value` (string or number), `role` (`button`, `text`,
-/// `element`), `traits`, `enabled`, `clickable`, `visible`, `frame`
-/// {x, y, width, height} and the activation point `x`, `y`. Frames and the
-/// activation point are in upright interface points, the space `input.tap`
-/// takes; SpringBoard's fixed-space frames are converted guest side.
-struct VPhoneUIInspectorElement: Identifiable {
- /// Position in the guest's AX order.
- let id: Int
- let role: String
- let label: String
- let identifier: String
- let value: String
- let traits: Int
- let isEnabled: Bool
- let isClickable: Bool
- let isVisible: Bool
- let frame: CGRect
- let tapPoint: CGPoint
- let details: [VPhoneUIInspectorDetail]
- let json: String
-
- init?(index: Int, object: [String: Any]) {
- guard let rect = object.object("frame"),
- let x = rect.double("x"), let y = rect.double("y"),
- let width = rect.double("width"), let height = rect.double("height")
- else { return nil }
- id = index
- role = object.string("role") ?? ""
- label = object.string("label") ?? ""
- identifier = object.string("identifier") ?? ""
- value = object["value"].map { value in
- (value as? String) ?? VPhoneUIInspectorRecord.display(value)
- } ?? ""
- traits = object.int("traits") ?? 0
- isEnabled = object.bool("enabled") ?? true
- isClickable = object.bool("clickable") ?? false
- isVisible = object.bool("visible") ?? true
- frame = CGRect(x: x, y: y, width: width, height: height)
- tapPoint = CGPoint(x: object.double("x") ?? frame.midX, y: object.double("y") ?? frame.midY)
- details = VPhoneUIInspectorRecord.details(object)
- json = VPhoneUIInspectorRecord.json(object)
- }
-
- // MARK: - Display
-
- var frameText: String {
- VPhoneUIInspectorRecord.frame(frame)
- }
-
- /// Reading order: top to bottom, then left to right.
- var frameOrder: Double {
- frame.minY * 100_000 + frame.minX
- }
-
- var clickableOrder: Int {
- isClickable ? 1 : 0
- }
-
- var enabledOrder: Int {
- isEnabled ? 1 : 0
- }
-
- /// The text a person would use to name this element.
- var summary: String {
- [label, identifier, value, role].first { !$0.isEmpty } ?? ""
- }
-}
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorModel.swift b/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorModel.swift
deleted file mode 100644
index 3d071281..00000000
--- a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorModel.swift
+++ /dev/null
@@ -1,408 +0,0 @@
-import AppKit
-import Foundation
-
-@MainActor
-@Observable
-final class VPhoneUIInspectorModel {
- // MARK: - Types
-
- enum Source: String, CaseIterable, Identifiable {
- case accessibility
- case text
-
- var id: Self {
- self
- }
-
- var title: String {
- switch self {
- case .accessibility: String(localized: "Accessibility", bundle: VPhoneLocalization.bundle)
- case .text: String(localized: "Text (OCR)", bundle: VPhoneLocalization.bundle)
- }
- }
- }
-
- enum Activity {
- case capturing
- case inspecting
- case recognizing
- case tapping
-
- var title: String {
- switch self {
- case .capturing: String(localized: "Capturing the guest screen…", bundle: VPhoneLocalization.bundle)
- case .inspecting: String(localized: "Reading accessibility elements…", bundle: VPhoneLocalization.bundle)
- case .recognizing: String(localized: "Recognizing text on the guest screen…", bundle: VPhoneLocalization.bundle)
- case .tapping: String(localized: "Tapping the guest…", bundle: VPhoneLocalization.bundle)
- }
- }
- }
-
- /// One outline drawn over the screenshot, in guest points.
- struct Overlay: Identifiable {
- let id: Int
- let frame: CGRect
- }
-
- /// The upper bound sent as `max_elements`; icli accepts 1…2000.
- static let maxElements = 1000
- /// Vision languages sent to `ui.ocr`.
- static let ocrLanguages = ["en-US", "zh-Hans"]
- static let ocrMinimumConfidence = 0.3
-
- // MARK: - State
-
- let control: VPhoneGuestControl
- var source: Source = .accessibility
- var visibleOnly = true
- var clickableOnly = false
-
- private(set) var activity: Activity?
- private(set) var status: VPhoneGuestToolStatus?
-
- private(set) var screenshot: NSImage?
- /// The screenshot's size in pixels.
- private(set) var screenshotPixels: CGSize?
- /// `device.screen` width and height, in points.
- private(set) var screenPoints: CGSize?
- private(set) var screenScale: Double?
-
- private(set) var appName = ""
- private(set) var appBundleID = ""
- private(set) var appPID: Int?
-
- private(set) var elements: [VPhoneUIInspectorElement] = []
- private(set) var elementsTruncated = false
- private(set) var hasLoadedElements = false
- var selectedElementID: VPhoneUIInspectorElement.ID?
- /// Guest AX order until a column header is clicked.
- var elementSortOrder = [KeyPathComparator(\VPhoneUIInspectorElement.id)]
-
- private(set) var textBlocks: [VPhoneUIInspectorTextBlock] = []
- private(set) var hasLoadedText = false
- var selectedTextID: VPhoneUIInspectorTextBlock.ID?
- var textSortOrder = [KeyPathComparator(\VPhoneUIInspectorTextBlock.frameOrder)]
-
- init(control: VPhoneGuestControl) {
- self.control = control
- }
-
- // MARK: - Derived
-
- var isBusy: Bool {
- activity != nil
- }
-
- var sortedElements: [VPhoneUIInspectorElement] {
- elements.sorted(using: elementSortOrder)
- }
-
- var sortedTextBlocks: [VPhoneUIInspectorTextBlock] {
- textBlocks.sorted(using: textSortOrder)
- }
-
- /// The coordinate space of frames and taps: guest points in the
- /// orientation of the screenshot. icli matches its OCR point space to the
- /// image the same way (`pointSizeMatching`).
- var pointSize: CGSize? {
- switch (screenPoints, screenshotPixels) {
- case let (points?, pixels?):
- let imageIsLandscape = pixels.width > pixels.height
- let screenIsLandscape = points.width > points.height
- return imageIsLandscape == screenIsLandscape
- ? points
- : CGSize(width: points.height, height: points.width)
- case let (points?, nil):
- return points
- case let (nil, pixels?):
- let scale = max(screenScale ?? 1, 1)
- return CGSize(width: pixels.width / scale, height: pixels.height / scale)
- case (nil, nil):
- return nil
- }
- }
-
- var overlays: [Overlay] {
- switch source {
- case .accessibility: elements.map { Overlay(id: $0.id, frame: $0.frame) }
- case .text: textBlocks.map { Overlay(id: $0.id, frame: $0.frame) }
- }
- }
-
- var selectedID: Int? {
- get {
- switch source {
- case .accessibility: selectedElementID
- case .text: selectedTextID
- }
- }
- set {
- switch source {
- case .accessibility: selectedElementID = newValue
- case .text: selectedTextID = newValue
- }
- }
- }
-
- var selectedElement: VPhoneUIInspectorElement? {
- selectedElementID.flatMap { id in elements.first { $0.id == id } }
- }
-
- var selectedTextBlock: VPhoneUIInspectorTextBlock? {
- selectedTextID.flatMap { id in textBlocks.first { $0.id == id } }
- }
-
- var selectedFrame: CGRect? {
- switch source {
- case .accessibility: selectedElement?.frame
- case .text: selectedTextBlock?.frame
- }
- }
-
- var selectedDetails: [VPhoneUIInspectorDetail] {
- switch source {
- case .accessibility: selectedElement?.details ?? []
- case .text: selectedTextBlock?.details ?? []
- }
- }
-
- var selectedJSON: String? {
- switch source {
- case .accessibility: selectedElement?.json
- case .text: selectedTextBlock?.json
- }
- }
-
- var selectedTapPoint: CGPoint? {
- switch source {
- case .accessibility: selectedElement?.tapPoint
- case .text: selectedTextBlock?.tapPoint
- }
- }
-
- var hasLoadedSource: Bool {
- switch source {
- case .accessibility: hasLoadedElements
- case .text: hasLoadedText
- }
- }
-
- var canTapSelected: Bool {
- selectedTapPoint != nil && control.isConnected && !isBusy
- }
-
- // MARK: - Hit Testing
-
- /// The smallest loaded frame of the current source containing `point`
- /// (guest points). Equal areas prefer the later element, which AX lists
- /// after its container.
- func hitTest(_ point: CGPoint) -> Int? {
- overlays
- .filter { $0.frame.width > 0 && $0.frame.height > 0 && $0.frame.contains(point) }
- .min { lhs, rhs in
- let lhsArea = lhs.frame.width * lhs.frame.height
- let rhsArea = rhs.frame.width * rhs.frame.height
- return lhsArea == rhsArea ? lhs.id > rhs.id : lhsArea < rhsArea
- }?
- .id
- }
-
- func select(at point: CGPoint) {
- selectedID = hitTest(point)
- }
-
- // MARK: - Loading
-
- /// Captures the screen, then reloads the current source.
- func refresh() async {
- guard activity == nil else { return }
- guard control.isConnected else {
- status = nil
- return
- }
- activity = .capturing
- defer { activity = nil }
- var failure: String?
-
- do {
- try await apply(screenshotJPEG: control.screenshotJPEG())
- } catch {
- failure = message(String(localized: "Unable to capture the guest screen. Check the connection, then try again.", bundle: VPhoneLocalization.bundle), error)
- }
- if let screen = try? await control.call("device.screen") {
- apply(screenResult: screen)
- }
- if let foreground = try? await control.call("apps.foreground") {
- apply(foregroundResult: foreground)
- }
-
- if let sourceFailure = await loadSource() {
- failure = failure ?? sourceFailure
- }
- if let failure {
- fail(failure)
- }
- }
-
- /// Reloads the current source without a new screenshot.
- func reloadSource() async {
- guard activity == nil, control.isConnected else { return }
- if let failure = await loadSource() {
- fail(failure)
- }
- activity = nil
- }
-
- /// Loads the current source; returns an error message on failure.
- private func loadSource() async -> String? {
- switch source {
- case .accessibility:
- activity = .inspecting
- do {
- try await apply(treeResult: control.call("ui.tree", params: [
- "max_elements": Self.maxElements,
- "visible_only": visibleOnly,
- "clickable_only": clickableOnly,
- ]))
- succeed(elementSummary)
- } catch {
- return message(String(localized: "Unable to read the accessibility elements. Bring an app to the front in the guest, then try again.", bundle: VPhoneLocalization.bundle), error)
- }
- case .text:
- activity = .recognizing
- do {
- try await apply(ocrResult: control.call("ui.ocr", params: [
- "languages": Self.ocrLanguages,
- "min_confidence": Self.ocrMinimumConfidence,
- ]))
- succeed(textSummary)
- } catch {
- return message(String(localized: "Unable to recognize text on the guest screen. Check the connection, then try again.", bundle: VPhoneLocalization.bundle), error)
- }
- }
- return nil
- }
-
- private var elementSummary: String {
- if elementsTruncated {
- return String(localized: "Showing the first \(elements.count) accessibility elements. The guest stopped reading at its limit.", bundle: VPhoneLocalization.bundle)
- }
- return elements.count == 1
- ? String(localized: "Found 1 accessibility element.", bundle: VPhoneLocalization.bundle)
- : String(localized: "Found \(elements.count) accessibility elements.", bundle: VPhoneLocalization.bundle)
- }
-
- private var textSummary: String {
- textBlocks.count == 1
- ? String(localized: "Recognized 1 text block.", bundle: VPhoneLocalization.bundle)
- : String(localized: "Recognized \(textBlocks.count) text blocks.", bundle: VPhoneLocalization.bundle)
- }
-
- // MARK: - Parsing
-
- func apply(screenshotJPEG data: Data) throws {
- guard let image = NSImage(data: data),
- let rep = image.representations.first,
- rep.pixelsWide > 0, rep.pixelsHigh > 0
- else {
- throw VPhoneGuestControl.ControlError.protocolError("invalid guest screenshot")
- }
- screenshot = image
- screenshotPixels = CGSize(width: rep.pixelsWide, height: rep.pixelsHigh)
- }
-
- /// `device.screen`: {width, height, scale, orientation, locked, screen_off}.
- func apply(screenResult result: [String: Any]) {
- if let width = result.double("width"), let height = result.double("height"), width > 0, height > 0 {
- screenPoints = CGSize(width: width, height: height)
- }
- screenScale = result.double("scale")
- }
-
- /// `apps.foreground`: {bundle_id, name, pid, verified, source}.
- func apply(foregroundResult result: [String: Any]) {
- appBundleID = result.string("bundle_id") ?? ""
- appName = result.string("name") ?? ""
- appPID = result.int("pid").flatMap { $0 > 0 ? $0 : nil }
- }
-
- /// `ui.tree`: {source, pid, elements, count, truncated}.
- func apply(treeResult result: [String: Any]) {
- let previous = selectedElement
- elements = result.objects("elements").enumerated().compactMap { index, object in
- VPhoneUIInspectorElement(index: index, object: object)
- }
- elementsTruncated = result.bool("truncated") ?? false
- hasLoadedElements = true
- selectedElementID = previous.flatMap { old in
- elements.first { $0.frame == old.frame && $0.summary == old.summary }?.id
- }
- }
-
- /// `ui.ocr`: {blocks, count, engine}.
- func apply(ocrResult result: [String: Any]) {
- let previous = selectedTextBlock
- textBlocks = result.objects("blocks").enumerated().compactMap { index, object in
- VPhoneUIInspectorTextBlock(index: index, object: object)
- }
- hasLoadedText = true
- selectedTextID = previous.flatMap { old in
- textBlocks.first { $0.text == old.text && $0.frame.intersects(old.frame) }?.id
- }
- }
-
- // MARK: - Actions
-
- func tapSelected() async {
- guard let point = selectedTapPoint else { return }
- await tap(at: point)
- }
-
- /// Taps `point` (guest points) on the guest, then refreshes once the
- /// guest has had a moment to react.
- func tap(at point: CGPoint) async {
- guard activity == nil, control.isConnected else { return }
- let bounds = pointSize ?? CGSize(width: point.x + 1, height: point.y + 1)
- let x = min(max(point.x, 0), max(bounds.width - 0.5, 0))
- let y = min(max(point.y, 0), max(bounds.height - 0.5, 0))
- activity = .tapping
- do {
- _ = try await control.call("input.tap", params: ["x": x, "y": y])
- } catch {
- activity = nil
- fail(message(String(localized: "Unable to tap the guest. Check the connection, then try again.", bundle: VPhoneLocalization.bundle), error))
- return
- }
- activity = nil
- try? await Task.sleep(for: .milliseconds(600))
- await refresh()
- }
-
- func copySelected() {
- guard let json = selectedJSON else { return }
- copy(json)
- succeed(String(localized: "Copied the selected element as JSON.", bundle: VPhoneLocalization.bundle))
- }
-
- func copy(_ text: String) {
- NSPasteboard.general.clearContents()
- NSPasteboard.general.setString(text, forType: .string)
- }
-
- // MARK: - Status
-
- private func message(_ base: String, _ error: any Error) -> String {
- guard case let VPhoneGuestControl.ControlError.guestError(detail) = error, !detail.isEmpty else {
- return base
- }
- return String(localized: "\(base) The guest reported: \(detail)", bundle: VPhoneLocalization.bundle)
- }
-
- private func succeed(_ message: String) {
- status = VPhoneGuestToolStatus(message: message, isError: false)
- }
-
- private func fail(_ message: String) {
- status = VPhoneGuestToolStatus(message: message, isError: true)
- }
-}
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorRecord.swift b/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorRecord.swift
deleted file mode 100644
index d60b717b..00000000
--- a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorRecord.swift
+++ /dev/null
@@ -1,78 +0,0 @@
-import Foundation
-
-// MARK: - Detail
-
-/// One key of a guest record, flattened for the detail readout
-/// (`frame.x`, `frame.width`, …).
-struct VPhoneUIInspectorDetail: Identifiable {
- let key: String
- let value: String
-
- var id: String {
- key
- }
-}
-
-// MARK: - Record Formatting
-
-/// Renders the raw vphoned dictionaries behind the inspector rows.
-enum VPhoneUIInspectorRecord {
- /// A point value without noise: `12`, `104.5`, `0.98`.
- static func number(_ value: Double, fractionDigits: Int = 2) -> String {
- guard value.isFinite else { return "—" }
- if value.rounded() == value, abs(value) < 1e15 {
- return String(Int(value))
- }
- var text = String(format: "%.*f", fractionDigits, value)
- while text.hasSuffix("0") {
- text.removeLast()
- }
- if text.hasSuffix(".") {
- text.removeLast()
- }
- return text
- }
-
- /// `x,y w×h` in guest points.
- static func frame(_ rect: CGRect) -> String {
- let point = { (value: Double) in number(value, fractionDigits: 1) }
- return "\(point(rect.minX)),\(point(rect.minY)) \(point(rect.width))×\(point(rect.height))"
- }
-
- static func details(_ object: [String: Any], prefix: String = "") -> [VPhoneUIInspectorDetail] {
- object.keys.sorted().flatMap { key -> [VPhoneUIInspectorDetail] in
- let path = prefix.isEmpty ? key : "\(prefix).\(key)"
- if let nested = object[key] as? [String: Any] {
- return details(nested, prefix: path)
- }
- return [VPhoneUIInspectorDetail(key: path, value: display(object[key]))]
- }
- }
-
- static func display(_ value: Any?) -> String {
- switch value {
- case let text as String:
- return text.isEmpty ? "\"\"" : text
- case let number as NSNumber:
- if CFGetTypeID(number) == CFBooleanGetTypeID() {
- return number.boolValue ? "true" : "false"
- }
- return self.number(number.doubleValue)
- case nil, is NSNull:
- return "null"
- case let other?:
- guard JSONSerialization.isValidJSONObject(other),
- let data = try? JSONSerialization.data(withJSONObject: other, options: [.sortedKeys])
- else { return String(describing: other) }
- return String(decoding: data, as: UTF8.self)
- }
- }
-
- static func json(_ object: [String: Any]) -> String {
- guard let data = try? JSONSerialization.data(
- withJSONObject: object,
- options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes],
- ) else { return "{}" }
- return String(decoding: data, as: UTF8.self)
- }
-}
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorScreenView.swift b/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorScreenView.swift
deleted file mode 100644
index b4323a6e..00000000
--- a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorScreenView.swift
+++ /dev/null
@@ -1,84 +0,0 @@
-import SwiftUI
-
-/// The guest screenshot with every loaded frame outlined. Click selects the
-/// smallest frame under the pointer; double-click taps that point on the guest.
-struct VPhoneUIInspectorScreenView: View {
- let model: VPhoneUIInspectorModel
-
- var body: some View {
- if let size = model.pointSize {
- screen(size: size)
- .aspectRatio(size, contentMode: .fit)
- .padding(12)
- .frame(maxWidth: .infinity, maxHeight: .infinity)
- } else if model.activity == .capturing {
- ProgressView()
- .frame(maxWidth: .infinity, maxHeight: .infinity)
- } else {
- VPhonePanelEmptyState(
- title: "No Screenshot",
- systemImage: "iphone",
- message: "Choose Refresh to capture the guest screen.",
- )
- }
- }
-
- // MARK: - Screen
-
- private func screen(size: CGSize) -> some View {
- ZStack {
- if let screenshot = model.screenshot {
- Image(nsImage: screenshot)
- .resizable()
- .interpolation(.high)
- } else {
- Rectangle()
- .fill(Color(nsColor: .textBackgroundColor))
- }
- GeometryReader { geometry in
- // Guest points to view points. The aspect ratio is fixed, so
- // one factor serves both axes.
- let scale = geometry.size.width / size.width
- overlay(scale: scale)
- .contentShape(Rectangle())
- .onTapGesture(count: 2, coordinateSpace: .local) { location in
- let point = CGPoint(x: location.x / scale, y: location.y / scale)
- model.select(at: point)
- Task { await model.tap(at: point) }
- }
- .simultaneousGesture(
- SpatialTapGesture(coordinateSpace: .local).onEnded { value in
- model.select(at: CGPoint(x: value.location.x / scale, y: value.location.y / scale))
- },
- )
- }
- }
- .overlay(Rectangle().stroke(Color(nsColor: .separatorColor), lineWidth: 1))
- .help("Click to select the element under the pointer. Double-click to tap there in the guest.")
- .accessibilityElement(children: .ignore)
- .accessibilityLabel("Guest screenshot")
- }
-
- private func overlay(scale: CGFloat) -> some View {
- let overlays = model.overlays
- let selected = model.selectedFrame
- let outline: Color = model.source == .accessibility ? .blue : .green
- return Canvas { context, _ in
- for item in overlays {
- let rect = scaled(item.frame, scale)
- guard rect.width > 0, rect.height > 0 else { continue }
- context.stroke(Path(rect), with: .color(outline.opacity(0.75)), lineWidth: 1)
- }
- if let selected {
- let rect = scaled(selected, scale)
- context.fill(Path(rect), with: .color(Color.accentColor.opacity(0.3)))
- context.stroke(Path(rect), with: .color(Color.accentColor), lineWidth: 2)
- }
- }
- }
-
- private func scaled(_ rect: CGRect, _ scale: CGFloat) -> CGRect {
- CGRect(x: rect.minX * scale, y: rect.minY * scale, width: rect.width * scale, height: rect.height * scale)
- .insetBy(dx: 0.5, dy: 0.5)
- }
-}
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorTextBlock.swift b/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorTextBlock.swift
deleted file mode 100644
index 1e0d8a57..00000000
--- a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorTextBlock.swift
+++ /dev/null
@@ -1,38 +0,0 @@
-import Foundation
-
-// MARK: - OCR Text Block
-
-/// One block of `ui.ocr`. icli reports `text`, `confidence` (0…1) and the
-/// block's center `x`, `y` plus `width`, `height`, all in screen points.
-struct VPhoneUIInspectorTextBlock: Identifiable {
- let id: Int
- let text: String
- let confidence: Double
- let frame: CGRect
- let details: [VPhoneUIInspectorDetail]
- let json: String
-
- init?(index: Int, object: [String: Any]) {
- guard let x = object.double("x"), let y = object.double("y"),
- let width = object.double("width"), let height = object.double("height")
- else { return nil }
- id = index
- text = object.string("text") ?? ""
- confidence = object.double("confidence") ?? 0
- frame = CGRect(x: x - width / 2, y: y - height / 2, width: width, height: height)
- details = VPhoneUIInspectorRecord.details(object)
- json = VPhoneUIInspectorRecord.json(object)
- }
-
- var frameText: String {
- VPhoneUIInspectorRecord.frame(frame)
- }
-
- var frameOrder: Double {
- frame.minY * 100_000 + frame.minX
- }
-
- var tapPoint: CGPoint {
- CGPoint(x: frame.midX, y: frame.midY)
- }
-}
diff --git a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorView.swift b/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorView.swift
deleted file mode 100644
index 03623a27..00000000
--- a/VPhoneExecutable/VPhoneVirtualization/UI/UserInterface/Panels/UIInspector/VPhoneUIInspectorView.swift
+++ /dev/null
@@ -1,321 +0,0 @@
-import SwiftUI
-
-struct VPhoneUIInspectorView: View {
- @Bindable var model: VPhoneUIInspectorModel
-
- var body: some View {
- VStack(spacing: 0) {
- header
- Divider()
- HSplitView {
- VPhoneUIInspectorScreenView(model: model)
- .frame(minWidth: 220, idealWidth: 320, maxWidth: 420, maxHeight: .infinity)
- recordsPane
- .frame(minWidth: 600, idealWidth: 720, maxWidth: .infinity, maxHeight: .infinity)
- }
- Divider()
- VPhoneGuestToolStatusBar(
- isConnected: model.control.isConnected,
- activity: model.activity?.title,
- status: model.status,
- )
- }
- .toolbar { toolbar }
- .guestToolShortcuts([
- VPhoneGuestToolShortcut(key: "r", isEnabled: !model.isBusy) {
- Task { await model.refresh() }
- },
- VPhoneGuestToolShortcut(key: "t", isEnabled: model.canTapSelected) {
- Task { await model.tapSelected() }
- },
- VPhoneGuestToolShortcut(key: "c", modifiers: [.command, .shift], isEnabled: model.selectedJSON != nil) {
- model.copySelected()
- },
- ])
- .task(id: model.control.isConnected) {
- if model.control.isConnected {
- await model.refresh()
- }
- }
- .onChange(of: model.source) { _, _ in
- if !model.hasLoadedSource {
- Task { await model.reloadSource() }
- }
- }
- .onChange(of: model.visibleOnly) { _, _ in
- Task { await model.reloadSource() }
- }
- .onChange(of: model.clickableOnly) { _, _ in
- Task { await model.reloadSource() }
- }
- }
-
- // MARK: - Toolbar
-
- @ToolbarContentBuilder
- private var toolbar: some ToolbarContent {
- ToolbarItemGroup(placement: .primaryAction) {
- Toggle("Visible Only", systemImage: "eye", isOn: $model.visibleOnly)
- .help("Show only accessibility elements inside the screen")
- .disabled(model.source != .accessibility || model.isBusy || !model.control.isConnected)
- Toggle("Clickable Only", systemImage: "hand.tap", isOn: $model.clickableOnly)
- .help("Show only accessibility elements that accept taps")
- .disabled(model.source != .accessibility || model.isBusy || !model.control.isConnected)
- Button("Tap Selected", systemImage: "hand.point.up.left") {
- Task { await model.tapSelected() }
- }
- .help("Tap the selected element in the guest (⌘T)")
- .disabled(!model.canTapSelected)
- Button("Copy", systemImage: "doc.on.doc") { model.copySelected() }
- .help("Copy the selected element as JSON (⇧⌘C)")
- .disabled(model.selectedJSON == nil)
- Button("Refresh", systemImage: "arrow.clockwise") { Task { await model.refresh() } }
- .help("Capture the guest screen and reload the elements (⌘R)")
- .disabled(model.isBusy || !model.control.isConnected)
- }
- }
-
- // MARK: - Header
-
- private var header: some View {
- HStack(spacing: 8) {
- Image(systemName: "app.dashed")
- .foregroundStyle(.secondary)
- if model.appBundleID.isEmpty {
- Text("Frontmost app unknown")
- .foregroundStyle(.secondary)
- } else {
- Text(model.appName.isEmpty ? model.appBundleID : model.appName)
- .fontWeight(.semibold)
- .lineLimit(1)
- VPhonePanelMonoText(model.appBundleID, secondary: true)
- if let pid = model.appPID {
- VPhonePanelMonoText("PID \(pid)", secondary: true)
- .fixedSize()
- }
- }
- Spacer(minLength: 8)
- if let points = model.pointSize {
- VPhonePanelMonoText(screenText(points), secondary: true)
- .fixedSize()
- }
- }
- .font(.system(size: 12))
- .padding(.horizontal, 12)
- .frame(height: 28)
- }
-
- private func screenText(_ points: CGSize) -> String {
- let size = "\(VPhoneUIInspectorRecord.number(points.width))×\(VPhoneUIInspectorRecord.number(points.height)) pt"
- guard let scale = model.screenScale else { return size }
- return "\(size) @\(VPhoneUIInspectorRecord.number(scale))x"
- }
-
- // MARK: - Records
-
- private var recordsPane: some View {
- VStack(spacing: 0) {
- HStack(spacing: 8) {
- Picker("Source", selection: $model.source) {
- ForEach(VPhoneUIInspectorModel.Source.allCases) { source in
- Text(source.title).tag(source)
- }
- }
- .pickerStyle(.segmented)
- .labelsHidden()
- .fixedSize()
- .help("Choose accessibility elements or text recognized in the screenshot")
- Spacer(minLength: 8)
- if model.hasLoadedSource {
- VPhonePanelMonoText(countText, secondary: true)
- .fixedSize()
- }
- }
- .padding(.horizontal, 12)
- .padding(.vertical, 8)
- Divider()
- records
- .frame(maxWidth: .infinity, maxHeight: .infinity)
- Divider()
- VPhoneUIInspectorDetailView(details: model.selectedDetails)
- .frame(height: 148)
- }
- }
-
- private var countText: String {
- switch model.source {
- case .accessibility:
- model.elementsTruncated
- ? String(localized: "\(model.elements.count)+ elements", bundle: VPhoneLocalization.bundle)
- : String(localized: "\(model.elements.count) elements", bundle: VPhoneLocalization.bundle)
- case .text:
- String(localized: "\(model.textBlocks.count) blocks", bundle: VPhoneLocalization.bundle)
- }
- }
-
- @ViewBuilder
- private var records: some View {
- switch model.source {
- case .accessibility:
- if !model.elements.isEmpty {
- elementTable
- } else if model.hasLoadedElements {
- VPhonePanelEmptyState(
- title: "No Elements",
- systemImage: "rectangle.dashed",
- message: "The frontmost app reported no accessibility elements. Turn off Visible Only or Clickable Only, or bring another app to the front.",
- )
- } else {
- placeholder
- }
- case .text:
- if !model.textBlocks.isEmpty {
- textTable
- } else if model.hasLoadedText {
- VPhonePanelEmptyState(
- title: "No Text",
- systemImage: "text.viewfinder",
- message: "No text was recognized on the guest screen.",
- )
- } else {
- placeholder
- }
- }
- }
-
- @ViewBuilder
- private var placeholder: some View {
- if model.isBusy {
- ProgressView()
- .frame(maxWidth: .infinity, maxHeight: .infinity)
- } else if model.control.isConnected {
- VPhonePanelEmptyState(
- title: "Not Loaded",
- systemImage: "arrow.clockwise",
- message: "Choose Refresh to inspect the guest screen.",
- )
- } else {
- VPhonePanelEmptyState(
- title: "Guest Not Connected",
- systemImage: "bolt.horizontal",
- message: "The inspector loads once vphoned is reachable in the guest.",
- )
- }
- }
-
- // MARK: - Accessibility Table
-
- private var elementTable: some View {
- ScrollViewReader { proxy in
- Table(model.sortedElements, selection: $model.selectedElementID, sortOrder: $model.elementSortOrder) {
- TableColumn("Role", value: \.role) { element in
- VPhonePanelMonoText(element.role)
- }
- .width(min: 52, ideal: 56, max: 90)
- TableColumn("Label", value: \.label) { element in
- VPhonePanelMonoText(element.label)
- }
- .width(min: 72, ideal: 96)
- TableColumn("Identifier", value: \.identifier) { element in
- VPhonePanelMonoText(element.identifier, secondary: true)
- }
- .width(min: 56, ideal: 60)
- TableColumn("Value", value: \.value) { element in
- VPhonePanelMonoText(element.value)
- }
- .width(min: 40, ideal: 44)
- TableColumn("Frame", value: \.frameOrder) { element in
- VPhonePanelMonoText(element.frameText)
- }
- .width(min: 104, ideal: 106, max: 170)
- TableColumn("Clickable", value: \.clickableOrder) { element in
- flag(element.isClickable, on: "Clickable", off: "Not clickable")
- }
- .width(min: 56, ideal: 56, max: 72)
- TableColumn("Enabled", value: \.enabledOrder) { element in
- flag(element.isEnabled, on: "Enabled", off: "Disabled", showsOff: true)
- }
- .width(min: 56, ideal: 56, max: 72)
- }
- .contextMenu(forSelectionType: VPhoneUIInspectorElement.ID.self) { ids in
- if let element = ids.first.flatMap({ id in model.elements.first { $0.id == id } }) {
- Button("Tap") { Task { await model.tap(at: element.tapPoint) } }
- .disabled(!model.control.isConnected || model.isBusy)
- Divider()
- Button("Copy as JSON") { model.copy(element.json) }
- Button("Copy Label") { model.copy(element.label) }
- .disabled(element.label.isEmpty)
- Button("Copy Identifier") { model.copy(element.identifier) }
- .disabled(element.identifier.isEmpty)
- Button("Copy Frame") { model.copy(element.frameText) }
- }
- } primaryAction: { ids in
- if let element = ids.first.flatMap({ id in model.elements.first { $0.id == id } }) {
- Task { await model.tap(at: element.tapPoint) }
- }
- }
- .onChange(of: model.selectedElementID) { _, id in
- if let id {
- proxy.scrollTo(id)
- }
- }
- }
- }
-
- private func flag(_ value: Bool, on: LocalizedStringKey, off: LocalizedStringKey, showsOff: Bool = false) -> some View {
- Group {
- if value {
- Image(systemName: "checkmark")
- .foregroundStyle(.secondary)
- } else if showsOff {
- Image(systemName: "xmark")
- .foregroundStyle(.orange)
- } else {
- Text(verbatim: "")
- }
- }
- .font(.system(size: 10, weight: .semibold))
- .frame(maxWidth: .infinity)
- .accessibilityLabel(Text(value ? on : off, bundle: VPhoneLocalization.bundle))
- }
-
- // MARK: - Text Table
-
- private var textTable: some View {
- ScrollViewReader { proxy in
- Table(model.sortedTextBlocks, selection: $model.selectedTextID, sortOrder: $model.textSortOrder) {
- TableColumn("Text", value: \.text) { block in
- VPhonePanelMonoText(block.text)
- }
- .width(min: 120, ideal: 220)
- TableColumn("Confidence", value: \.confidence) { block in
- VPhonePanelMonoText(VPhonePanelFormat.percent(block.confidence), secondary: block.confidence < 0.5)
- }
- .width(min: 84, ideal: 88, max: 110)
- TableColumn("Frame", value: \.frameOrder) { block in
- VPhonePanelMonoText(block.frameText)
- }
- .width(min: 104, ideal: 120, max: 170)
- }
- .contextMenu(forSelectionType: VPhoneUIInspectorTextBlock.ID.self) { ids in
- if let block = ids.first.flatMap({ id in model.textBlocks.first { $0.id == id } }) {
- Button("Tap") { Task { await model.tap(at: block.tapPoint) } }
- .disabled(!model.control.isConnected || model.isBusy)
- Divider()
- Button("Copy as JSON") { model.copy(block.json) }
- Button("Copy Text") { model.copy(block.text) }
- Button("Copy Frame") { model.copy(block.frameText) }
- }
- } primaryAction: { ids in
- if let block = ids.first.flatMap({ id in model.textBlocks.first { $0.id == id } }) {
- Task { await model.tap(at: block.tapPoint) }
- }
- }
- .onChange(of: model.selectedTextID) { _, id in
- if let id {
- proxy.scrollTo(id)
- }
- }
- }
- }
-}